Note: This is test shard 8 of 8.
[==========] Running 7 tests from 6 test suites.
[----------] Global test environment set-up.
[----------] 1 test from TableRangeRebalancingTest
[ RUN ] TableRangeRebalancingTest.Basic
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240501 08:10:41.952610 28678 test_util.cc:268] Using random seed: 97389367
I20240501 08:10:41.963658 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:10:41.963737 28678 ts_itest-base.cc:113] --------------
I20240501 08:10:41.963779 28678 ts_itest-base.cc:114] 3 tablet servers
I20240501 08:10:41.963836 28678 ts_itest-base.cc:115] 3 replicas per TS
I20240501 08:10:41.963901 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:10:41Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:10:41Z Disabled control of system clock
I20240501 08:10:41.985105 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:42809
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:34443
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:42809
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240501 08:10:42.302054 28690 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:42.302364 28690 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:42.302453 28690 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:42.310240 28690 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:42.310370 28690 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:10:42.310417 28690 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:42.310472 28690 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:10:42.310516 28690 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:10:42.319489 28690 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:34443
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:42809
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:42809
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:42.320781 28690 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:42.322875 28690 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:42.333194 28695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:42.333827 28698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:42.334019 28696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:42.334616 28690 server_base.cc:1034] running on GCE node
I20240501 08:10:42.335357 28690 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:42.336761 28690 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:42.337970 28690 hybrid_clock.cc:648] HybridClock initialized: now 1714551042337907 us; error 48 us; skew 500 ppm
I20240501 08:10:42.338305 28690 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:42.340564 28690 webserver.cc:460] Webserver started at http://127.28.1.190:35559/ using document root <none> and password file <none>
I20240501 08:10:42.341289 28690 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:42.341395 28690 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:42.341711 28690 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:42.344303 28690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "a519c8b1af2f4fe797d563b4c19f7300"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.345131 28690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "a519c8b1af2f4fe797d563b4c19f7300"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.350255 28690 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:42.353458 28704 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:42.354701 28690 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240501 08:10:42.354858 28690 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "a519c8b1af2f4fe797d563b4c19f7300"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.355039 28690 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:42.382288 28690 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:42.383121 28690 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:42.383378 28690 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:42.401547 28690 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:42809
I20240501 08:10:42.401556 28755 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:42809 every 8 connection(s)
I20240501 08:10:42.402808 28690 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:10:42.406354 28756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:42.409073 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 28690
I20240501 08:10:42.409344 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:10:42.418391 28756 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300: Bootstrap starting.
I20240501 08:10:42.422173 28756 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:42.423420 28756 log.cc:824] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:42.426465 28756 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300: No bootstrap required, opened a new log
I20240501 08:10:42.434582 28756 raft_consensus.cc:348] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } }
I20240501 08:10:42.434996 28756 raft_consensus.cc:374] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:42.435110 28756 raft_consensus.cc:729] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a519c8b1af2f4fe797d563b4c19f7300, State: Initialized, Role: FOLLOWER
I20240501 08:10:42.435722 28756 consensus_queue.cc:260] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } }
I20240501 08:10:42.436390 28756 raft_consensus.cc:388] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:10:42.436532 28756 raft_consensus.cc:482] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:10:42.436677 28756 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:42.439035 28756 raft_consensus.cc:504] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } }
I20240501 08:10:42.439518 28756 leader_election.cc:302] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: a519c8b1af2f4fe797d563b4c19f7300; no voters:
I20240501 08:10:42.440160 28756 leader_election.cc:288] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:10:42.440289 28761 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:42.442198 28756 sys_catalog.cc:564] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:10:42.444314 28761 raft_consensus.cc:686] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [term 1 LEADER]: Becoming Leader. State: Replica: a519c8b1af2f4fe797d563b4c19f7300, State: Running, Role: LEADER
I20240501 08:10:42.444994 28761 consensus_queue.cc:237] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } }
I20240501 08:10:42.447978 28763 sys_catalog.cc:455] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a519c8b1af2f4fe797d563b4c19f7300. Latest consensus state: current_term: 1 leader_uuid: "a519c8b1af2f4fe797d563b4c19f7300" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } } }
I20240501 08:10:42.448287 28763 sys_catalog.cc:458] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:42.448166 28762 sys_catalog.cc:455] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "a519c8b1af2f4fe797d563b4c19f7300" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a519c8b1af2f4fe797d563b4c19f7300" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 42809 } } }
I20240501 08:10:42.448443 28762 sys_catalog.cc:458] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300 [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:42.452602 28774 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:10:42.456552 28774 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:10:42.463989 28774 catalog_manager.cc:1347] Generated new cluster ID: 17dda8c58eb948aca2162c2fc16c7e8b
I20240501 08:10:42.464107 28774 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:10:42.480564 28774 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:10:42.481647 28774 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:10:42.492641 28774 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a519c8b1af2f4fe797d563b4c19f7300: Generated new TSK 0
I20240501 08:10:42.493422 28774 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:10:42.504782 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--builtin_ntp_servers=127.28.1.148:34443
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:10:42.834192 28780 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:42.834486 28780 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:42.834614 28780 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:42.842677 28780 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:42.842833 28780 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:42.842986 28780 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:10:42.852528 28780 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:34443
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:42.853828 28780 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:42.855506 28780 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:42.867717 28785 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:42.868281 28788 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:42.868660 28786 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:42.869202 28780 server_base.cc:1034] running on GCE node
I20240501 08:10:42.869850 28780 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:42.870833 28780 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:42.872058 28780 hybrid_clock.cc:648] HybridClock initialized: now 1714551042872009 us; error 43 us; skew 500 ppm
I20240501 08:10:42.872385 28780 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:42.874572 28780 webserver.cc:460] Webserver started at http://127.28.1.129:39803/ using document root <none> and password file <none>
I20240501 08:10:42.875304 28780 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:42.875416 28780 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:42.875741 28780 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:42.878391 28780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "cd25c86f47a44a01ac0d1a2be410088d"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.879158 28780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "cd25c86f47a44a01ac0d1a2be410088d"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.885155 28780 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
I20240501 08:10:42.888597 28794 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:42.890043 28780 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:10:42.890224 28780 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "cd25c86f47a44a01ac0d1a2be410088d"
format_stamp: "Formatted at 2024-05-01 08:10:42 on dist-test-slave-jj4n"
I20240501 08:10:42.890425 28780 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:42.905887 28780 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:42.906756 28780 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:42.907032 28780 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:42.908071 28780 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:42.909751 28780 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:42.909852 28780 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:42.909960 28780 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:42.910027 28780 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:42.941066 28780 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:43033
I20240501 08:10:42.941110 28906 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:43033 every 8 connection(s)
I20240501 08:10:42.942451 28780 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:10:42.950220 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 28780
I20240501 08:10:42.950423 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:10:42.953972 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--builtin_ntp_servers=127.28.1.148:34443
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:42.955859 28907 heartbeater.cc:347] Connected to a master server at 127.28.1.190:42809
I20240501 08:10:42.956173 28907 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:42.956967 28907 heartbeater.cc:511] Master 127.28.1.190:42809 requested a full tablet report, sending...
I20240501 08:10:42.959010 28721 ts_manager.cc:194] Registered new tserver with Master: cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033)
I20240501 08:10:42.960685 28721 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:59805
W20240501 08:10:43.275645 28911 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:43.275940 28911 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:43.276074 28911 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:43.283722 28911 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:43.283850 28911 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:43.284003 28911 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:10:43.293414 28911 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:34443
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:43.294653 28911 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:43.296368 28911 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:43.307497 28917 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:43.307669 28916 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:43.308876 28919 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:43.309509 28911 server_base.cc:1034] running on GCE node
I20240501 08:10:43.310062 28911 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:43.310941 28911 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:43.312115 28911 hybrid_clock.cc:648] HybridClock initialized: now 1714551043312063 us; error 45 us; skew 500 ppm
I20240501 08:10:43.312410 28911 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:43.317488 28911 webserver.cc:460] Webserver started at http://127.28.1.130:36591/ using document root <none> and password file <none>
I20240501 08:10:43.318205 28911 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:43.318297 28911 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:43.318590 28911 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:43.321095 28911 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "9c47b857624d401cb096a1b82023b8a6"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.321807 28911 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "9c47b857624d401cb096a1b82023b8a6"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.326632 28911 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:43.329842 28925 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.331130 28911 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:10:43.331282 28911 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "9c47b857624d401cb096a1b82023b8a6"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.331459 28911 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:43.352912 28911 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:43.353709 28911 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:43.353976 28911 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:43.354887 28911 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:43.356333 28911 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:43.356416 28911 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.356515 28911 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:43.356587 28911 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.388931 28911 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:45849
I20240501 08:10:43.388975 29037 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:45849 every 8 connection(s)
I20240501 08:10:43.390201 28911 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:10:43.399673 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 28911
I20240501 08:10:43.399842 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:10:43.402477 29038 heartbeater.cc:347] Connected to a master server at 127.28.1.190:42809
I20240501 08:10:43.402778 29038 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:43.403295 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--builtin_ntp_servers=127.28.1.148:34443
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:43.403626 29038 heartbeater.cc:511] Master 127.28.1.190:42809 requested a full tablet report, sending...
I20240501 08:10:43.404978 28721 ts_manager.cc:194] Registered new tserver with Master: 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:43.405722 28721 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:53399
W20240501 08:10:43.726505 29041 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:43.726807 29041 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:43.726943 29041 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:43.734120 29041 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:43.734253 29041 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:43.734406 29041 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:10:43.743494 29041 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:34443
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:42809
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:43.744808 29041 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:43.746495 29041 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:43.758188 29046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:43.758503 29049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:43.758847 29047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:43.759660 29041 server_base.cc:1034] running on GCE node
I20240501 08:10:43.760295 29041 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:43.761166 29041 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:43.762360 29041 hybrid_clock.cc:648] HybridClock initialized: now 1714551043762306 us; error 47 us; skew 500 ppm
I20240501 08:10:43.762668 29041 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:43.764729 29041 webserver.cc:460] Webserver started at http://127.28.1.131:37447/ using document root <none> and password file <none>
I20240501 08:10:43.765406 29041 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:43.765508 29041 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:43.765808 29041 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:43.768257 29041 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "643a571dd31d4f31b73e1cd53416c561"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.768949 29041 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "643a571dd31d4f31b73e1cd53416c561"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.774078 29041 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
I20240501 08:10:43.777264 29055 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.778618 29041 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:10:43.778750 29041 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "643a571dd31d4f31b73e1cd53416c561"
format_stamp: "Formatted at 2024-05-01 08:10:43 on dist-test-slave-jj4n"
I20240501 08:10:43.778920 29041 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:43.820824 29041 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:43.821674 29041 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:43.821920 29041 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:43.822819 29041 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:43.824277 29041 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:43.824362 29041 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.824442 29041 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:43.824513 29041 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:43.855304 29041 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:36895
I20240501 08:10:43.855383 29167 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:36895 every 8 connection(s)
I20240501 08:10:43.856585 29041 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:10:43.861410 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 29041
I20240501 08:10:43.861578 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:10:43.869451 29168 heartbeater.cc:347] Connected to a master server at 127.28.1.190:42809
I20240501 08:10:43.869741 29168 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:43.870657 29168 heartbeater.cc:511] Master 127.28.1.190:42809 requested a full tablet report, sending...
I20240501 08:10:43.871945 28721 ts_manager.cc:194] Registered new tserver with Master: 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:43.872751 28721 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:37733
I20240501 08:10:43.877913 28678 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240501 08:10:43.897420 28678 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240501 08:10:43.964088 28907 heartbeater.cc:503] Master 127.28.1.190:42809 was elected leader, sending a full tablet report...
I20240501 08:10:44.408110 29038 heartbeater.cc:503] Master 127.28.1.190:42809 was elected leader, sending a full tablet report...
I20240501 08:10:44.875017 29168 heartbeater.cc:503] Master 127.28.1.190:42809 was elected leader, sending a full tablet report...
I20240501 08:10:47.651432 28721 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59150:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
rows: "<redacted>""\006\001\000\000\000\000\200\007\001\000\000\000\000\000\006\001\000\000\000\000\000\007\001\000\377\377\377\177"
indirect_data: "<redacted>"""
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 8
seed: 0
}
}
W20240501 08:10:47.657750 28721 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240501 08:10:47.704249 28963 tablet_service.cc:1467] Processing CreateTablet for tablet f2cf4e6a8bc346c791ac7b64d823e5bb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.703696 28962 tablet_service.cc:1467] Processing CreateTablet for tablet af919c096bf845b287bc8eb84ab4c266 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.704608 28960 tablet_service.cc:1467] Processing CreateTablet for tablet 8724bf810a21401a91ecab6cb9d07588 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.705556 28967 tablet_service.cc:1467] Processing CreateTablet for tablet 91f59b211cef4cdb87dbc95a78fc412b (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.704879 28961 tablet_service.cc:1467] Processing CreateTablet for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.705297 28972 tablet_service.cc:1467] Processing CreateTablet for tablet 72fb544bcc4643398ce932f884afdfbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.705509 28959 tablet_service.cc:1467] Processing CreateTablet for tablet 309b6208e40e4ea398071b1920f147e1 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.703043 28973 tablet_service.cc:1467] Processing CreateTablet for tablet 9c5cb3ebcb0943018df9dd29ab313c40 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.703303 28966 tablet_service.cc:1467] Processing CreateTablet for tablet dd52aada8a994e7a9a98aca87a0377b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.707130 28960 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8724bf810a21401a91ecab6cb9d07588. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.704584 28969 tablet_service.cc:1467] Processing CreateTablet for tablet 250d9bebc71d4a76957b22897562faca (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.704931 28971 tablet_service.cc:1467] Processing CreateTablet for tablet c14911bdc3b3444a9a07bc58f7d4ea83 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.710489 28967 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91f59b211cef4cdb87dbc95a78fc412b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.710117 28839 tablet_service.cc:1467] Processing CreateTablet for tablet 49d6e21f117b4738aafa3d7ebaffabdb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.705246 28965 tablet_service.cc:1467] Processing CreateTablet for tablet 00f202cb3ace4e75983d5e82f6470943 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.711642 28965 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00f202cb3ace4e75983d5e82f6470943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.712407 28837 tablet_service.cc:1467] Processing CreateTablet for tablet b7c75235244d42e1859a7998ff02967d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.713668 28835 tablet_service.cc:1467] Processing CreateTablet for tablet dd52aada8a994e7a9a98aca87a0377b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.714438 28833 tablet_service.cc:1467] Processing CreateTablet for tablet b56cc1f96fd94a15930eef89d48e1e45 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.713979 29103 tablet_service.cc:1467] Processing CreateTablet for tablet 9c5cb3ebcb0943018df9dd29ab313c40 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.715140 28831 tablet_service.cc:1467] Processing CreateTablet for tablet af919c096bf845b287bc8eb84ab4c266 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.715788 29103 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c5cb3ebcb0943018df9dd29ab313c40. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.715875 28829 tablet_service.cc:1467] Processing CreateTablet for tablet 8724bf810a21401a91ecab6cb9d07588 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.705426 28964 tablet_service.cc:1467] Processing CreateTablet for tablet b56cc1f96fd94a15930eef89d48e1e45 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.716265 28961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.716480 28964 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b56cc1f96fd94a15930eef89d48e1e45. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.716563 28827 tablet_service.cc:1467] Processing CreateTablet for tablet 787f5278de3245c9bebf92be0155a5d9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.717366 29102 tablet_service.cc:1467] Processing CreateTablet for tablet 72fb544bcc4643398ce932f884afdfbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.718152 29100 tablet_service.cc:1467] Processing CreateTablet for tablet 49d6e21f117b4738aafa3d7ebaffabdb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.703778 28968 tablet_service.cc:1467] Processing CreateTablet for tablet b7c75235244d42e1859a7998ff02967d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.718755 28840 tablet_service.cc:1467] Processing CreateTablet for tablet c14911bdc3b3444a9a07bc58f7d4ea83 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.718966 28968 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b7c75235244d42e1859a7998ff02967d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.719430 28838 tablet_service.cc:1467] Processing CreateTablet for tablet 250d9bebc71d4a76957b22897562faca (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.720124 29101 tablet_service.cc:1467] Processing CreateTablet for tablet c14911bdc3b3444a9a07bc58f7d4ea83 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.720134 28836 tablet_service.cc:1467] Processing CreateTablet for tablet 91f59b211cef4cdb87dbc95a78fc412b (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.721318 28834 tablet_service.cc:1467] Processing CreateTablet for tablet 00f202cb3ace4e75983d5e82f6470943 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.721447 28972 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72fb544bcc4643398ce932f884afdfbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.722003 28832 tablet_service.cc:1467] Processing CreateTablet for tablet f2cf4e6a8bc346c791ac7b64d823e5bb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.703228 28970 tablet_service.cc:1467] Processing CreateTablet for tablet 49d6e21f117b4738aafa3d7ebaffabdb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.722702 28828 tablet_service.cc:1467] Processing CreateTablet for tablet 309b6208e40e4ea398071b1920f147e1 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.723110 28970 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 49d6e21f117b4738aafa3d7ebaffabdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.723371 28830 tablet_service.cc:1467] Processing CreateTablet for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.705700 28958 tablet_service.cc:1467] Processing CreateTablet for tablet 787f5278de3245c9bebf92be0155a5d9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.724318 28959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 309b6208e40e4ea398071b1920f147e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.724673 28958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 787f5278de3245c9bebf92be0155a5d9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.725041 28962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af919c096bf845b287bc8eb84ab4c266. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.725157 28841 tablet_service.cc:1467] Processing CreateTablet for tablet 72fb544bcc4643398ce932f884afdfbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.726492 28842 tablet_service.cc:1467] Processing CreateTablet for tablet 9c5cb3ebcb0943018df9dd29ab313c40 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.711928 28839 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 49d6e21f117b4738aafa3d7ebaffabdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.727397 29099 tablet_service.cc:1467] Processing CreateTablet for tablet 250d9bebc71d4a76957b22897562faca (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.727738 28836 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91f59b211cef4cdb87dbc95a78fc412b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.728003 28842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c5cb3ebcb0943018df9dd29ab313c40. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.728250 28833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b56cc1f96fd94a15930eef89d48e1e45. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.728546 29098 tablet_service.cc:1467] Processing CreateTablet for tablet b7c75235244d42e1859a7998ff02967d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.728729 28831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af919c096bf845b287bc8eb84ab4c266. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.729123 28838 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 250d9bebc71d4a76957b22897562faca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.729354 28829 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8724bf810a21401a91ecab6cb9d07588. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.729480 29097 tablet_service.cc:1467] Processing CreateTablet for tablet 91f59b211cef4cdb87dbc95a78fc412b (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.729581 28832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2cf4e6a8bc346c791ac7b64d823e5bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.730026 28827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 787f5278de3245c9bebf92be0155a5d9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.730278 28840 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c14911bdc3b3444a9a07bc58f7d4ea83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.730396 29096 tablet_service.cc:1467] Processing CreateTablet for tablet dd52aada8a994e7a9a98aca87a0377b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.730731 28837 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b7c75235244d42e1859a7998ff02967d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.731559 29095 tablet_service.cc:1467] Processing CreateTablet for tablet 00f202cb3ace4e75983d5e82f6470943 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.731798 28841 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72fb544bcc4643398ce932f884afdfbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.733254 29094 tablet_service.cc:1467] Processing CreateTablet for tablet b56cc1f96fd94a15930eef89d48e1e45 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.735541 28835 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd52aada8a994e7a9a98aca87a0377b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.736183 28830 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.739583 28834 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00f202cb3ace4e75983d5e82f6470943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.739930 29093 tablet_service.cc:1467] Processing CreateTablet for tablet f2cf4e6a8bc346c791ac7b64d823e5bb (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.742353 28966 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd52aada8a994e7a9a98aca87a0377b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.742831 28828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 309b6208e40e4ea398071b1920f147e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.742789 29092 tablet_service.cc:1467] Processing CreateTablet for tablet af919c096bf845b287bc8eb84ab4c266 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.748302 28971 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c14911bdc3b3444a9a07bc58f7d4ea83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.750391 29102 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72fb544bcc4643398ce932f884afdfbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.750741 29097 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91f59b211cef4cdb87dbc95a78fc412b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.750978 29096 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd52aada8a994e7a9a98aca87a0377b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.750833 29091 tablet_service.cc:1467] Processing CreateTablet for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.751350 29091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d33c53a8cc2491a8b23a2a05bd9b26d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.751686 29090 tablet_service.cc:1467] Processing CreateTablet for tablet 8724bf810a21401a91ecab6cb9d07588 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.752293 29089 tablet_service.cc:1467] Processing CreateTablet for tablet 309b6208e40e4ea398071b1920f147e1 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240501 08:10:47.752552 29093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2cf4e6a8bc346c791ac7b64d823e5bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.752758 29098 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b7c75235244d42e1859a7998ff02967d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.753001 29090 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8724bf810a21401a91ecab6cb9d07588. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.753219 29094 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b56cc1f96fd94a15930eef89d48e1e45. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.753468 29099 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 250d9bebc71d4a76957b22897562faca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.753484 29088 tablet_service.cc:1467] Processing CreateTablet for tablet 787f5278de3245c9bebf92be0155a5d9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=30991a50a70a48e8ad7bc8495a24f17b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240501 08:10:47.754190 28969 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 250d9bebc71d4a76957b22897562faca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.754422 29100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 49d6e21f117b4738aafa3d7ebaffabdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.755061 29088 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 787f5278de3245c9bebf92be0155a5d9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.755544 29089 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 309b6208e40e4ea398071b1920f147e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.756464 29101 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c14911bdc3b3444a9a07bc58f7d4ea83. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.757233 29092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af919c096bf845b287bc8eb84ab4c266. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.765695 29095 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00f202cb3ace4e75983d5e82f6470943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.776499 28963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2cf4e6a8bc346c791ac7b64d823e5bb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.808707 28973 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c5cb3ebcb0943018df9dd29ab313c40. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:47.827836 29186 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.835489 29186 tablet_bootstrap.cc:654] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.836997 29186 log.cc:824] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:47.843145 29187 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.843220 29188 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.845675 29186 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.845983 29186 ts_tablet_manager.cc:1367] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.018s user 0.009s sys 0.004s
I20240501 08:10:47.846359 29188 tablet_bootstrap.cc:654] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.846522 29187 tablet_bootstrap.cc:654] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.847344 29188 log.cc:824] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:47.847695 29187 log.cc:824] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:47.850392 29187 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.850625 29188 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.850638 29187 ts_tablet_manager.cc:1367] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20240501 08:10:47.850911 29188 ts_tablet_manager.cc:1367] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20240501 08:10:47.855005 29186 raft_consensus.cc:348] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.855542 29186 raft_consensus.cc:374] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.855638 29186 raft_consensus.cc:729] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.856487 29186 consensus_queue.cc:260] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.856927 29187 raft_consensus.cc:348] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.857323 29187 raft_consensus.cc:374] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.857421 29187 raft_consensus.cc:729] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.858163 29187 consensus_queue.cc:260] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.860122 29188 raft_consensus.cc:348] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.860599 29188 raft_consensus.cc:374] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.860723 29188 raft_consensus.cc:729] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.861493 29188 consensus_queue.cc:260] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.866402 29186 ts_tablet_manager.cc:1398] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.020s user 0.005s sys 0.014s
I20240501 08:10:47.869853 29188 ts_tablet_manager.cc:1398] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.019s user 0.018s sys 0.001s
I20240501 08:10:47.870288 29188 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.872025 29188 tablet_bootstrap.cc:654] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.874276 29187 ts_tablet_manager.cc:1398] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.024s user 0.008s sys 0.014s
I20240501 08:10:47.874673 29187 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.876039 29186 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.876549 29187 tablet_bootstrap.cc:654] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.877777 29186 tablet_bootstrap.cc:654] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.879947 29188 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.880154 29188 ts_tablet_manager.cc:1367] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
I20240501 08:10:47.880414 29186 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.880551 29186 ts_tablet_manager.cc:1367] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:10:47.880658 29187 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.880805 29187 ts_tablet_manager.cc:1367] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.001s
I20240501 08:10:47.881265 29188 raft_consensus.cc:348] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.881485 29188 raft_consensus.cc:374] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.881394 29186 raft_consensus.cc:348] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.881572 29188 raft_consensus.cc:729] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.881604 29186 raft_consensus.cc:374] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.881526 29187 raft_consensus.cc:348] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.881680 29186 raft_consensus.cc:729] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.881717 29187 raft_consensus.cc:374] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.881796 29187 raft_consensus.cc:729] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.881825 29188 consensus_queue.cc:260] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.882017 29187 consensus_queue.cc:260] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.881954 29186 consensus_queue.cc:260] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.882524 29187 ts_tablet_manager.cc:1398] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.882705 29186 ts_tablet_manager.cc:1398] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.882784 29187 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.882946 29186 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.884486 29187 tablet_bootstrap.cc:654] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.884547 29186 tablet_bootstrap.cc:654] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.884282 29188 ts_tablet_manager.cc:1398] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.888113 29188 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.890635 29188 tablet_bootstrap.cc:654] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.892802 29188 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.892966 29188 ts_tablet_manager.cc:1367] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:10:47.893354 29187 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.893539 29187 ts_tablet_manager.cc:1367] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.003s
I20240501 08:10:47.893553 29188 raft_consensus.cc:348] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.893795 29188 raft_consensus.cc:374] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.893870 29188 raft_consensus.cc:729] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.893918 29186 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.894120 29188 consensus_queue.cc:260] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
W20240501 08:10:47.894315 29039 tablet.cc:2369] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:10:47.894640 29188 ts_tablet_manager.cc:1398] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.894551 29187 raft_consensus.cc:348] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.894774 29187 raft_consensus.cc:374] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.894840 29187 raft_consensus.cc:729] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.894975 29188 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.895198 29186 ts_tablet_manager.cc:1367] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.012s user 0.005s sys 0.006s
I20240501 08:10:47.895176 29187 consensus_queue.cc:260] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.895705 29187 ts_tablet_manager.cc:1398] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.896003 29187 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.895885 29186 raft_consensus.cc:348] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.896129 29186 raft_consensus.cc:374] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.896204 29186 raft_consensus.cc:729] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.896533 29188 tablet_bootstrap.cc:654] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.897831 29187 tablet_bootstrap.cc:654] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.896490 29186 consensus_queue.cc:260] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.900462 29192 raft_consensus.cc:482] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:47.900709 29192 raft_consensus.cc:504] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.903187 29192 leader_election.cc:288] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:47.906831 29187 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.907027 29187 ts_tablet_manager.cc:1367] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.007s
I20240501 08:10:47.907850 29187 raft_consensus.cc:348] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.908111 29187 raft_consensus.cc:374] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.908205 29187 raft_consensus.cc:729] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.908447 29187 consensus_queue.cc:260] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.909052 29187 ts_tablet_manager.cc:1398] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.909366 29187 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.911065 29186 ts_tablet_manager.cc:1398] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.016s user 0.002s sys 0.000s
I20240501 08:10:47.911155 29187 tablet_bootstrap.cc:654] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.911433 29186 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.912829 29188 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.913007 29188 ts_tablet_manager.cc:1367] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.018s user 0.006s sys 0.004s
I20240501 08:10:47.913653 29186 tablet_bootstrap.cc:654] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.913735 29188 raft_consensus.cc:348] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.913947 29188 raft_consensus.cc:374] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.914026 29188 raft_consensus.cc:729] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.914255 29188 consensus_queue.cc:260] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.914863 29188 ts_tablet_manager.cc:1398] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.915190 29188 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.916914 29188 tablet_bootstrap.cc:654] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.921270 29186 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.921459 29186 ts_tablet_manager.cc:1367] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.005s
I20240501 08:10:47.921806 29187 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.921976 29187 ts_tablet_manager.cc:1367] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.000s
I20240501 08:10:47.922165 29186 raft_consensus.cc:348] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.922384 29186 raft_consensus.cc:374] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.922472 29186 raft_consensus.cc:729] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.922683 29186 consensus_queue.cc:260] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.922721 29187 raft_consensus.cc:348] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.922899 29187 raft_consensus.cc:374] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.922977 29187 raft_consensus.cc:729] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.923197 29186 ts_tablet_manager.cc:1398] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.923223 29187 consensus_queue.cc:260] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.923461 29186 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.923700 29187 ts_tablet_manager.cc:1398] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.924037 29187 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.925149 29186 tablet_bootstrap.cc:654] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.925552 29187 tablet_bootstrap.cc:654] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.927248 29188 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.927435 29188 ts_tablet_manager.cc:1367] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.003s
I20240501 08:10:47.928262 29188 raft_consensus.cc:348] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.928467 29188 raft_consensus.cc:374] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.928534 29188 raft_consensus.cc:729] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.928820 29188 consensus_queue.cc:260] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.929402 29188 ts_tablet_manager.cc:1398] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.929709 29188 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.930194 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:47.931203 29188 tablet_bootstrap.cc:654] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
W20240501 08:10:47.931897 28927 leader_election.cc:341] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:10:47.934669 29186 tablet_bootstrap.cc:492] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.934847 29186 ts_tablet_manager.cc:1367] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.000s
I20240501 08:10:47.936403 29187 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.936558 29187 ts_tablet_manager.cc:1367] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.008s
I20240501 08:10:47.936476 29186 raft_consensus.cc:348] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.937182 29186 raft_consensus.cc:374] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.937330 29186 raft_consensus.cc:729] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.937366 29187 raft_consensus.cc:348] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.937551 29187 raft_consensus.cc:374] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.937631 29187 raft_consensus.cc:729] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.937637 29186 consensus_queue.cc:260] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.937875 29187 consensus_queue.cc:260] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.938138 29186 ts_tablet_manager.cc:1398] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:10:47.938351 29187 ts_tablet_manager.cc:1398] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.938393 29186 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.938616 29187 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.939661 29186 tablet_bootstrap.cc:654] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.940129 29187 tablet_bootstrap.cc:654] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.941145 29188 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.941341 29188 ts_tablet_manager.cc:1367] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.008s
I20240501 08:10:47.942071 29188 raft_consensus.cc:348] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.942258 29188 raft_consensus.cc:374] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.942351 29188 raft_consensus.cc:729] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.942610 29188 consensus_queue.cc:260] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.943217 29188 ts_tablet_manager.cc:1398] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.943502 29188 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.943394 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:47.945897 29186 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.946048 29186 ts_tablet_manager.cc:1367] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.002s
W20240501 08:10:47.946041 28928 leader_election.cc:341] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:10:47.946220 28928 leader_election.cc:302] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6; no voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d
I20240501 08:10:47.946687 29192 raft_consensus.cc:2726] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:10:47.946898 29186 raft_consensus.cc:348] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.947075 29186 raft_consensus.cc:374] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.947156 29186 raft_consensus.cc:729] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.947363 29186 consensus_queue.cc:260] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.947736 29186 ts_tablet_manager.cc:1398] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
W20240501 08:10:47.947988 28908 tablet.cc:2369] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:10:47.948035 29186 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.949364 29186 tablet_bootstrap.cc:654] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.950358 29188 tablet_bootstrap.cc:654] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.952244 29188 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.952250 29186 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.952390 29188 ts_tablet_manager.cc:1367] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240501 08:10:47.952399 29186 ts_tablet_manager.cc:1367] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:10:47.952947 29186 raft_consensus.cc:348] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.953140 29186 raft_consensus.cc:374] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.953212 29186 raft_consensus.cc:729] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.953423 29186 consensus_queue.cc:260] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.953603 29188 raft_consensus.cc:348] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.953819 29188 raft_consensus.cc:374] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.953878 29188 raft_consensus.cc:729] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.953974 29186 ts_tablet_manager.cc:1398] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240501 08:10:47.954115 29188 consensus_queue.cc:260] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.954237 29186 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.954584 29188 ts_tablet_manager.cc:1398] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.954854 29188 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.955945 29186 tablet_bootstrap.cc:654] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.956437 29188 tablet_bootstrap.cc:654] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.957881 29187 tablet_bootstrap.cc:492] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.958066 29187 ts_tablet_manager.cc:1367] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.019s user 0.000s sys 0.006s
I20240501 08:10:47.958845 29187 raft_consensus.cc:348] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.959098 29187 raft_consensus.cc:374] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.959160 29187 raft_consensus.cc:729] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.959429 29187 consensus_queue.cc:260] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.960206 29187 ts_tablet_manager.cc:1398] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.960520 29187 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.962225 29187 tablet_bootstrap.cc:654] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.964193 29186 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.964339 29186 ts_tablet_manager.cc:1367] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.007s
I20240501 08:10:47.964783 29188 tablet_bootstrap.cc:492] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.964934 29188 ts_tablet_manager.cc:1367] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.008s
I20240501 08:10:47.965101 29186 raft_consensus.cc:348] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.965291 29186 raft_consensus.cc:374] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.965385 29186 raft_consensus.cc:729] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.965580 29186 consensus_queue.cc:260] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.965755 29188 raft_consensus.cc:348] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.965950 29188 raft_consensus.cc:374] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.966023 29188 raft_consensus.cc:729] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.966194 29186 ts_tablet_manager.cc:1398] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.966454 29186 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.966269 29188 consensus_queue.cc:260] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.966837 29188 ts_tablet_manager.cc:1398] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.968083 29186 tablet_bootstrap.cc:654] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.969313 29187 tablet_bootstrap.cc:492] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.969460 29187 ts_tablet_manager.cc:1367] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.009s user 0.005s sys 0.001s
I20240501 08:10:47.970187 29187 raft_consensus.cc:348] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.970386 29187 raft_consensus.cc:374] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.970494 29187 raft_consensus.cc:729] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.970722 29187 consensus_queue.cc:260] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.971433 29188 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.971848 29186 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.972011 29186 ts_tablet_manager.cc:1367] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.003s
I20240501 08:10:47.972617 29186 raft_consensus.cc:348] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.972832 29186 raft_consensus.cc:374] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.972904 29186 raft_consensus.cc:729] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.973064 29188 tablet_bootstrap.cc:654] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.973194 29187 ts_tablet_manager.cc:1398] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240501 08:10:47.973127 29186 consensus_queue.cc:260] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.973676 29186 ts_tablet_manager.cc:1398] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:10:47.973986 29186 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.975410 29186 tablet_bootstrap.cc:654] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.977872 29186 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.977998 29186 ts_tablet_manager.cc:1367] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:10:47.978654 29186 raft_consensus.cc:348] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.978801 29187 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.978844 29186 raft_consensus.cc:374] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.978928 29186 raft_consensus.cc:729] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.979138 29186 consensus_queue.cc:260] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.979773 29186 ts_tablet_manager.cc:1398] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.980049 29186 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.981999 29186 tablet_bootstrap.cc:654] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.983820 29188 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.983951 29186 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.984114 29186 ts_tablet_manager.cc:1367] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.001s
I20240501 08:10:47.984210 29187 tablet_bootstrap.cc:654] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.984156 29188 ts_tablet_manager.cc:1367] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.008s
I20240501 08:10:47.984843 29186 raft_consensus.cc:348] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.985069 29186 raft_consensus.cc:374] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.984953 29188 raft_consensus.cc:348] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.985141 29186 raft_consensus.cc:729] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.985221 29188 raft_consensus.cc:374] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.985289 29188 raft_consensus.cc:729] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.985358 29186 consensus_queue.cc:260] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.985527 29188 consensus_queue.cc:260] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.986014 29188 ts_tablet_manager.cc:1398] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:47.986074 29186 ts_tablet_manager.cc:1398] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:10:47.986311 29186 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.986328 29188 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.986649 29187 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.986805 29187 ts_tablet_manager.cc:1367] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240501 08:10:47.987844 29186 tablet_bootstrap.cc:654] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.987756 29187 raft_consensus.cc:348] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.988006 29187 raft_consensus.cc:374] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.988083 29187 raft_consensus.cc:729] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.988291 29188 tablet_bootstrap.cc:654] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.988350 29187 consensus_queue.cc:260] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.988818 29187 ts_tablet_manager.cc:1398] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:47.989113 29187 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:47.990571 29187 tablet_bootstrap.cc:654] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.995792 29186 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:47.995934 29186 ts_tablet_manager.cc:1367] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.001s
I20240501 08:10:47.996467 29188 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:47.996590 29188 ts_tablet_manager.cc:1367] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.010s user 0.007s sys 0.002s
I20240501 08:10:47.996624 29187 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:47.996829 29187 ts_tablet_manager.cc:1367] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20240501 08:10:47.996729 29186 raft_consensus.cc:348] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.996925 29186 raft_consensus.cc:374] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.997017 29186 raft_consensus.cc:729] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.997274 29186 consensus_queue.cc:260] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.997769 29188 raft_consensus.cc:348] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.997892 29186 ts_tablet_manager.cc:1398] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:10:47.997931 29188 raft_consensus.cc:374] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:47.998025 29188 raft_consensus.cc:729] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:47.998143 29186 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:47.998215 29188 consensus_queue.cc:260] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:47.998677 29188 ts_tablet_manager.cc:1398] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:47.998903 29188 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:47.999662 29186 tablet_bootstrap.cc:654] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.000159 29188 tablet_bootstrap.cc:654] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:47.997582 29187 raft_consensus.cc:348] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.000542 29187 raft_consensus.cc:374] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.000675 29187 raft_consensus.cc:729] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.002553 29187 consensus_queue.cc:260] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.003198 29187 ts_tablet_manager.cc:1398] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240501 08:10:48.003547 29187 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.005121 29187 tablet_bootstrap.cc:654] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.005391 29188 tablet_bootstrap.cc:492] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.005545 29188 ts_tablet_manager.cc:1367] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:10:48.006824 29188 raft_consensus.cc:348] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.007056 29188 raft_consensus.cc:374] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.007128 29188 raft_consensus.cc:729] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.007355 29188 consensus_queue.cc:260] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.007872 29188 ts_tablet_manager.cc:1398] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:10:48.008176 29188 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:48.009768 29188 tablet_bootstrap.cc:654] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.011785 29186 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:48.011938 29186 ts_tablet_manager.cc:1367] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.014s user 0.001s sys 0.011s
I20240501 08:10:48.012704 29186 raft_consensus.cc:348] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.012961 29186 raft_consensus.cc:374] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.013044 29186 raft_consensus.cc:729] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.013063 29188 tablet_bootstrap.cc:492] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.013128 29187 tablet_bootstrap.cc:492] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.013185 29188 ts_tablet_manager.cc:1367] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:10:48.013285 29187 ts_tablet_manager.cc:1367] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20240501 08:10:48.013319 29186 consensus_queue.cc:260] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.013829 29186 ts_tablet_manager.cc:1398] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:10:48.013885 29188 raft_consensus.cc:348] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.014055 29188 raft_consensus.cc:374] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.014004 29187 raft_consensus.cc:348] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.014135 29188 raft_consensus.cc:729] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.014223 29186 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:48.014202 29187 raft_consensus.cc:374] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.014333 29187 raft_consensus.cc:729] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.014335 29188 consensus_queue.cc:260] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.014709 29187 consensus_queue.cc:260] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.016100 29187 ts_tablet_manager.cc:1398] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:10:48.016351 29186 tablet_bootstrap.cc:654] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.016367 29187 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.016942 29188 ts_tablet_manager.cc:1398] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:10:48.017863 29187 tablet_bootstrap.cc:654] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.017983 29188 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:48.019557 29188 tablet_bootstrap.cc:654] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.023747 29186 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:48.023891 29186 ts_tablet_manager.cc:1367] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20240501 08:10:48.024103 29187 tablet_bootstrap.cc:492] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.024236 29187 ts_tablet_manager.cc:1367] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20240501 08:10:48.024530 29188 tablet_bootstrap.cc:492] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.024722 29188 ts_tablet_manager.cc:1367] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:10:48.024679 29186 raft_consensus.cc:348] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.024907 29186 raft_consensus.cc:374] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.024868 29187 raft_consensus.cc:348] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.024996 29186 raft_consensus.cc:729] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.025056 29187 raft_consensus.cc:374] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.025143 29187 raft_consensus.cc:729] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.025264 29186 consensus_queue.cc:260] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.025403 29188 raft_consensus.cc:348] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.025610 29188 raft_consensus.cc:374] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.025679 29188 raft_consensus.cc:729] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.025846 29188 consensus_queue.cc:260] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.025358 29187 consensus_queue.cc:260] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.026369 29186 ts_tablet_manager.cc:1398] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:10:48.026363 29188 ts_tablet_manager.cc:1398] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:48.026625 29186 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:48.026675 29188 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:48.028069 29188 tablet_bootstrap.cc:654] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.028082 29186 tablet_bootstrap.cc:654] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.028653 29187 ts_tablet_manager.cc:1398] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:10:48.029662 29187 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.031136 29187 tablet_bootstrap.cc:654] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.036396 29187 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.036573 29187 ts_tablet_manager.cc:1367] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:10:48.036604 29186 tablet_bootstrap.cc:492] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:48.036746 29186 ts_tablet_manager.cc:1367] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
I20240501 08:10:48.036818 29188 tablet_bootstrap.cc:492] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.036964 29188 ts_tablet_manager.cc:1367] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.004s
I20240501 08:10:48.037377 29187 raft_consensus.cc:348] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.037384 29186 raft_consensus.cc:348] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.037609 29186 raft_consensus.cc:374] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.037609 29187 raft_consensus.cc:374] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.037511 29188 raft_consensus.cc:348] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.037694 29187 raft_consensus.cc:729] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.037698 29186 raft_consensus.cc:729] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.037706 29188 raft_consensus.cc:374] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.037787 29188 raft_consensus.cc:729] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.037935 29186 consensus_queue.cc:260] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.037938 29187 consensus_queue.cc:260] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.038017 29188 consensus_queue.cc:260] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.038589 29188 ts_tablet_manager.cc:1398] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:10:48.038833 29188 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:48.040266 29186 ts_tablet_manager.cc:1398] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:10:48.040292 29188 tablet_bootstrap.cc:654] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.040565 29186 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6: Bootstrap starting.
I20240501 08:10:48.041455 29187 ts_tablet_manager.cc:1398] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240501 08:10:48.042683 29186 tablet_bootstrap.cc:654] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.044140 29187 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.044528 29192 raft_consensus.cc:482] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.045286 29192 raft_consensus.cc:504] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.045884 29187 tablet_bootstrap.cc:654] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.046020 29192 leader_election.cc:288] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.046849 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd52aada8a994e7a9a98aca87a0377b4" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.047261 29123 raft_consensus.cc:2454] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.047627 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd52aada8a994e7a9a98aca87a0377b4" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.047677 28927 leader_election.cc:302] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.047891 29187 tablet_bootstrap.cc:492] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.047926 29192 raft_consensus.cc:2781] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.047955 28862 raft_consensus.cc:2454] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.048043 29187 ts_tablet_manager.cc:1367] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:10:48.048064 29192 raft_consensus.cc:482] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.048144 29192 raft_consensus.cc:3037] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.048560 29187 raft_consensus.cc:348] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.048724 29187 raft_consensus.cc:374] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.048803 29187 raft_consensus.cc:729] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.049021 29187 consensus_queue.cc:260] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.049261 29186 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6: No bootstrap required, opened a new log
I20240501 08:10:48.049404 29186 ts_tablet_manager.cc:1367] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.001s
I20240501 08:10:48.050171 29186 raft_consensus.cc:348] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.050374 29186 raft_consensus.cc:374] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.050315 29192 raft_consensus.cc:504] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.050462 29186 raft_consensus.cc:729] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.050509 29187 ts_tablet_manager.cc:1398] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:10:48.050750 29187 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.050696 29186 consensus_queue.cc:260] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.051035 29192 leader_election.cc:288] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.052246 29188 tablet_bootstrap.cc:492] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.052392 29187 tablet_bootstrap.cc:654] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.052495 29188 ts_tablet_manager.cc:1367] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.014s user 0.005s sys 0.004s
I20240501 08:10:48.053213 29188 raft_consensus.cc:348] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.053509 29188 raft_consensus.cc:374] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.053628 29188 raft_consensus.cc:729] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.053895 29188 consensus_queue.cc:260] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.054335 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd52aada8a994e7a9a98aca87a0377b4" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.054594 29188 ts_tablet_manager.cc:1398] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:48.054904 29188 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d: Bootstrap starting.
I20240501 08:10:48.055583 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd52aada8a994e7a9a98aca87a0377b4" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.055799 29123 raft_consensus.cc:3037] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.056597 29188 tablet_bootstrap.cc:654] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.059219 29192 raft_consensus.cc:482] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.059374 29192 raft_consensus.cc:504] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.060124 29192 leader_election.cc:288] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.060813 29122 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8724bf810a21401a91ecab6cb9d07588" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.061157 29122 raft_consensus.cc:2454] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.061286 28861 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8724bf810a21401a91ecab6cb9d07588" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.061581 28861 raft_consensus.cc:2454] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.061622 28927 leader_election.cc:302] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.061942 29192 raft_consensus.cc:2781] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.062049 29192 raft_consensus.cc:482] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.062137 29192 raft_consensus.cc:3037] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.054535 28862 raft_consensus.cc:3037] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.064188 29188 tablet_bootstrap.cc:492] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d: No bootstrap required, opened a new log
I20240501 08:10:48.064255 29187 tablet_bootstrap.cc:492] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.064339 29188 ts_tablet_manager.cc:1367] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.004s
I20240501 08:10:48.064416 29187 ts_tablet_manager.cc:1367] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.014s user 0.012s sys 0.000s
I20240501 08:10:48.065109 29187 raft_consensus.cc:348] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.065129 29188 raft_consensus.cc:348] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.065212 29192 raft_consensus.cc:504] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.065318 29187 raft_consensus.cc:374] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.065336 29188 raft_consensus.cc:374] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.065378 29187 raft_consensus.cc:729] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.065416 29188 raft_consensus.cc:729] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.065622 29187 consensus_queue.cc:260] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.065630 29188 consensus_queue.cc:260] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.065986 29192 leader_election.cc:288] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.066251 29122 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8724bf810a21401a91ecab6cb9d07588" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.066427 29122 raft_consensus.cc:3037] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.066438 29188 ts_tablet_manager.cc:1398] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:48.066942 28861 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8724bf810a21401a91ecab6cb9d07588" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.067090 28861 raft_consensus.cc:3037] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.069100 29122 raft_consensus.cc:2454] T 8724bf810a21401a91ecab6cb9d07588 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.069132 29187 ts_tablet_manager.cc:1398] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:10:48.069481 29187 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561: Bootstrap starting.
I20240501 08:10:48.070025 28927 leader_election.cc:302] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.070325 29192 raft_consensus.cc:2781] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.070472 29192 raft_consensus.cc:686] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [term 1 LEADER]: Becoming Leader. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Running, Role: LEADER
I20240501 08:10:48.071010 29192 consensus_queue.cc:237] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.071308 29187 tablet_bootstrap.cc:654] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:48.068441 28862 raft_consensus.cc:2454] T dd52aada8a994e7a9a98aca87a0377b4 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.072270 28928 leader_election.cc:302] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.070662 28861 raft_consensus.cc:2454] T 8724bf810a21401a91ecab6cb9d07588 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.073148 29243 raft_consensus.cc:2781] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.073546 29123 raft_consensus.cc:2454] T dd52aada8a994e7a9a98aca87a0377b4 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.073737 29243 raft_consensus.cc:686] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [term 1 LEADER]: Becoming Leader. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Running, Role: LEADER
I20240501 08:10:48.074067 29243 consensus_queue.cc:237] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.075381 29186 ts_tablet_manager.cc:1398] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6: Time spent starting tablet: real 0.026s user 0.006s sys 0.005s
I20240501 08:10:48.077919 29192 raft_consensus.cc:482] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.078085 29192 raft_consensus.cc:504] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.078709 29192 leader_election.cc:288] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.079046 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c5cb3ebcb0943018df9dd29ab313c40" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.079219 29123 raft_consensus.cc:2454] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.078786 28721 catalog_manager.cc:5484] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "9c47b857624d401cb096a1b82023b8a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.079581 28927 leader_election.cc:302] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.079862 29192 raft_consensus.cc:2781] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.079916 29187 tablet_bootstrap.cc:492] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561: No bootstrap required, opened a new log
I20240501 08:10:48.079957 29192 raft_consensus.cc:482] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.080066 29187 ts_tablet_manager.cc:1367] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.003s
I20240501 08:10:48.080089 29192 raft_consensus.cc:3037] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.080462 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c5cb3ebcb0943018df9dd29ab313c40" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.080648 28862 raft_consensus.cc:2454] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.080862 29193 raft_consensus.cc:482] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.081005 29193 raft_consensus.cc:504] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.082495 29192 raft_consensus.cc:504] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.083292 29192 leader_election.cc:288] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.082862 29193 leader_election.cc:288] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.083775 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c5cb3ebcb0943018df9dd29ab313c40" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.084014 28862 raft_consensus.cc:3037] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.086962 28862 raft_consensus.cc:2454] T 9c5cb3ebcb0943018df9dd29ab313c40 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.087313 29187 raft_consensus.cc:348] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.087656 29187 raft_consensus.cc:374] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:48.087796 29187 raft_consensus.cc:729] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Initialized, Role: FOLLOWER
I20240501 08:10:48.088135 29187 consensus_queue.cc:260] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.088342 28928 leader_election.cc:302] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.088922 29187 ts_tablet_manager.cc:1398] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240501 08:10:48.090603 29192 raft_consensus.cc:2781] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.090808 29192 raft_consensus.cc:686] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [term 1 LEADER]: Becoming Leader. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Running, Role: LEADER
I20240501 08:10:48.091162 29192 consensus_queue.cc:237] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.094059 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c5cb3ebcb0943018df9dd29ab313c40" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.094276 29123 raft_consensus.cc:3037] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.096295 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "49d6e21f117b4738aafa3d7ebaffabdb" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.096557 28862 raft_consensus.cc:2454] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.096712 28720 catalog_manager.cc:5484] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "9c47b857624d401cb096a1b82023b8a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.097308 28720 catalog_manager.cc:5484] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "9c47b857624d401cb096a1b82023b8a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.097546 29123 raft_consensus.cc:2454] T 9c5cb3ebcb0943018df9dd29ab313c40 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.098441 29058 leader_election.cc:302] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.098868 29193 raft_consensus.cc:2781] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.098995 29193 raft_consensus.cc:482] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.099093 29193 raft_consensus.cc:3037] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.101810 29193 raft_consensus.cc:504] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.102205 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "49d6e21f117b4738aafa3d7ebaffabdb" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.102422 29193 leader_election.cc:288] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.102640 28993 raft_consensus.cc:2454] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.102960 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "49d6e21f117b4738aafa3d7ebaffabdb" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.103199 28992 raft_consensus.cc:3037] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.103245 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "49d6e21f117b4738aafa3d7ebaffabdb" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.103463 28862 raft_consensus.cc:3037] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.105252 28862 raft_consensus.cc:2454] T 49d6e21f117b4738aafa3d7ebaffabdb P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.105587 29058 leader_election.cc:302] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.105701 28992 raft_consensus.cc:2454] T 49d6e21f117b4738aafa3d7ebaffabdb P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.105885 29193 raft_consensus.cc:2781] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.106278 29193 raft_consensus.cc:686] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [term 1 LEADER]: Becoming Leader. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Running, Role: LEADER
I20240501 08:10:48.106766 29193 consensus_queue.cc:237] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.107024 29249 raft_consensus.cc:482] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.107151 29249 raft_consensus.cc:504] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.108122 29249 leader_election.cc:288] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.108170 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72fb544bcc4643398ce932f884afdfbc" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.108192 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72fb544bcc4643398ce932f884afdfbc" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.108342 28992 raft_consensus.cc:2454] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.108445 28862 raft_consensus.cc:2454] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.108704 29056 leader_election.cc:302] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.109057 29249 raft_consensus.cc:2781] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.109164 29249 raft_consensus.cc:482] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.109252 29249 raft_consensus.cc:3037] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.111244 29249 raft_consensus.cc:504] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
W20240501 08:10:48.111500 29169 tablet.cc:2369] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:10:48.111490 28720 catalog_manager.cc:5484] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 reported cstate change: term changed from 0 to 1, leader changed from <none> to 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "643a571dd31d4f31b73e1cd53416c561" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: HEALTHY } } }
I20240501 08:10:48.112270 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72fb544bcc4643398ce932f884afdfbc" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.112462 28992 raft_consensus.cc:3037] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.113268 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72fb544bcc4643398ce932f884afdfbc" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.113445 28862 raft_consensus.cc:3037] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.114914 28992 raft_consensus.cc:2454] T 72fb544bcc4643398ce932f884afdfbc P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.115146 29249 leader_election.cc:288] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.115260 29056 leader_election.cc:302] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.115475 29249 raft_consensus.cc:2781] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.115592 29249 raft_consensus.cc:686] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [term 1 LEADER]: Becoming Leader. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Running, Role: LEADER
I20240501 08:10:48.115864 29249 consensus_queue.cc:237] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.116495 28862 raft_consensus.cc:2454] T 72fb544bcc4643398ce932f884afdfbc P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.118573 28720 catalog_manager.cc:5484] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 reported cstate change: term changed from 0 to 1, leader changed from <none> to 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "643a571dd31d4f31b73e1cd53416c561" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: HEALTHY } } }
I20240501 08:10:48.122925 29192 raft_consensus.cc:482] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.123075 29192 raft_consensus.cc:504] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.123647 29192 leader_election.cc:288] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.124069 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2cf4e6a8bc346c791ac7b64d823e5bb" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.124100 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2cf4e6a8bc346c791ac7b64d823e5bb" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.124294 28862 raft_consensus.cc:2454] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.124294 29123 raft_consensus.cc:2454] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 0.
I20240501 08:10:48.124636 28927 leader_election.cc:302] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.124876 29192 raft_consensus.cc:2781] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.124990 29192 raft_consensus.cc:482] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.125075 29192 raft_consensus.cc:3037] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.126786 29192 raft_consensus.cc:504] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.127317 29192 leader_election.cc:288] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.127565 29194 raft_consensus.cc:482] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.127776 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2cf4e6a8bc346c791ac7b64d823e5bb" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.127717 29194 raft_consensus.cc:504] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.127916 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2cf4e6a8bc346c791ac7b64d823e5bb" candidate_uuid: "9c47b857624d401cb096a1b82023b8a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.128011 28862 raft_consensus.cc:3037] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.128142 29123 raft_consensus.cc:3037] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.129752 29194 leader_election.cc:288] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.129987 29123 raft_consensus.cc:2454] T f2cf4e6a8bc346c791ac7b64d823e5bb P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.130620 28862 raft_consensus.cc:2454] T f2cf4e6a8bc346c791ac7b64d823e5bb P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c47b857624d401cb096a1b82023b8a6 in term 1.
I20240501 08:10:48.131047 28928 leader_election.cc:302] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.131480 29192 raft_consensus.cc:2781] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.131628 29192 raft_consensus.cc:686] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [term 1 LEADER]: Becoming Leader. State: Replica: 9c47b857624d401cb096a1b82023b8a6, State: Running, Role: LEADER
I20240501 08:10:48.131907 29192 consensus_queue.cc:237] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.140048 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b7c75235244d42e1859a7998ff02967d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.140314 28992 raft_consensus.cc:2454] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.140981 28795 leader_election.cc:302] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.141410 29194 raft_consensus.cc:2781] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.141528 29194 raft_consensus.cc:482] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.141607 29194 raft_consensus.cc:3037] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.142515 28720 catalog_manager.cc:5484] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "9c47b857624d401cb096a1b82023b8a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.143381 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b7c75235244d42e1859a7998ff02967d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.143600 29123 raft_consensus.cc:2454] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.144934 29194 raft_consensus.cc:504] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.145638 29194 leader_election.cc:288] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.146009 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b7c75235244d42e1859a7998ff02967d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.146042 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b7c75235244d42e1859a7998ff02967d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.146162 29123 raft_consensus.cc:3037] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.146239 28992 raft_consensus.cc:3037] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.148098 28992 raft_consensus.cc:2454] T b7c75235244d42e1859a7998ff02967d P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.148101 29123 raft_consensus.cc:2454] T b7c75235244d42e1859a7998ff02967d P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.148445 28796 leader_election.cc:302] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.148726 29194 raft_consensus.cc:2781] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.149109 29194 raft_consensus.cc:686] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.149582 29194 consensus_queue.cc:237] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.153754 28720 catalog_manager.cc:5484] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.154173 29194 raft_consensus.cc:482] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.154302 29194 raft_consensus.cc:504] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.154842 29194 leader_election.cc:288] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.155256 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91f59b211cef4cdb87dbc95a78fc412b" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.155412 29249 raft_consensus.cc:482] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.155498 28992 raft_consensus.cc:2454] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.155582 29249 raft_consensus.cc:504] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.155795 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91f59b211cef4cdb87dbc95a78fc412b" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.155895 28795 leader_election.cc:302] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.156028 29123 raft_consensus.cc:2454] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.156302 29249 leader_election.cc:288] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.156551 29194 raft_consensus.cc:2781] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.156638 29194 raft_consensus.cc:482] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.156888 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "309b6208e40e4ea398071b1920f147e1" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.156970 29194 raft_consensus.cc:3037] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.157076 28992 raft_consensus.cc:2454] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.157002 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "309b6208e40e4ea398071b1920f147e1" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.157320 28862 raft_consensus.cc:2454] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.157402 29056 leader_election.cc:302] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.157657 29249 raft_consensus.cc:2781] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.157752 29249 raft_consensus.cc:482] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.157861 29249 raft_consensus.cc:3037] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.159691 29194 raft_consensus.cc:504] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.160373 29194 leader_election.cc:288] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.160825 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91f59b211cef4cdb87dbc95a78fc412b" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.160904 29249 raft_consensus.cc:504] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.161049 29123 raft_consensus.cc:3037] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.160740 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91f59b211cef4cdb87dbc95a78fc412b" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.161558 29249 leader_election.cc:288] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.162312 28992 raft_consensus.cc:3037] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.162397 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "309b6208e40e4ea398071b1920f147e1" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.162570 28993 raft_consensus.cc:3037] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.164610 28993 raft_consensus.cc:2454] T 309b6208e40e4ea398071b1920f147e1 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.164713 29123 raft_consensus.cc:2454] T 91f59b211cef4cdb87dbc95a78fc412b P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.164959 29056 leader_election.cc:302] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.165102 28992 raft_consensus.cc:2454] T 91f59b211cef4cdb87dbc95a78fc412b P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.165192 29249 raft_consensus.cc:2781] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.165349 29249 raft_consensus.cc:686] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [term 1 LEADER]: Becoming Leader. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Running, Role: LEADER
I20240501 08:10:48.165386 28796 leader_election.cc:302] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.165643 29194 raft_consensus.cc:2781] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.165809 29194 raft_consensus.cc:686] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.165616 29249 consensus_queue.cc:237] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.166097 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "309b6208e40e4ea398071b1920f147e1" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.166044 29194 consensus_queue.cc:237] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.166270 28862 raft_consensus.cc:3037] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.168779 28862 raft_consensus.cc:2454] T 309b6208e40e4ea398071b1920f147e1 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.168965 28720 catalog_manager.cc:5484] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.169656 28721 catalog_manager.cc:5484] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 reported cstate change: term changed from 0 to 1, leader changed from <none> to 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "643a571dd31d4f31b73e1cd53416c561" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: HEALTHY } } }
I20240501 08:10:48.177695 29249 raft_consensus.cc:482] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.177908 29249 raft_consensus.cc:504] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.179087 29249 leader_election.cc:288] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.179659 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "250d9bebc71d4a76957b22897562faca" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.179638 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "250d9bebc71d4a76957b22897562faca" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.179894 28992 raft_consensus.cc:2454] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.179900 28862 raft_consensus.cc:2454] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.180346 29056 leader_election.cc:302] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.180613 29249 raft_consensus.cc:2781] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.180729 29249 raft_consensus.cc:482] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.180811 29249 raft_consensus.cc:3037] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.182683 29249 raft_consensus.cc:504] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.183238 29249 leader_election.cc:288] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Requested vote from peers cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033), 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849)
I20240501 08:10:48.183763 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "250d9bebc71d4a76957b22897562faca" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.183773 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "250d9bebc71d4a76957b22897562faca" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.183992 28992 raft_consensus.cc:3037] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.184019 28862 raft_consensus.cc:3037] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.186750 28862 raft_consensus.cc:2454] T 250d9bebc71d4a76957b22897562faca P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.186758 28992 raft_consensus.cc:2454] T 250d9bebc71d4a76957b22897562faca P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.187192 29058 leader_election.cc:302] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.187494 29249 raft_consensus.cc:2781] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.187645 29249 raft_consensus.cc:686] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [term 1 LEADER]: Becoming Leader. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Running, Role: LEADER
I20240501 08:10:48.187939 29249 consensus_queue.cc:237] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.189424 29194 raft_consensus.cc:482] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.189558 29194 raft_consensus.cc:504] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.189841 29255 raft_consensus.cc:482] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.189982 29255 raft_consensus.cc:504] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.190256 29194 leader_election.cc:288] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.190645 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c14911bdc3b3444a9a07bc58f7d4ea83" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.190774 29255 leader_election.cc:288] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.190797 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c14911bdc3b3444a9a07bc58f7d4ea83" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.190886 28992 raft_consensus.cc:2454] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.191028 29123 raft_consensus.cc:2454] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.191431 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b56cc1f96fd94a15930eef89d48e1e45" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.191473 28795 leader_election.cc:302] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.191629 28992 raft_consensus.cc:2454] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.191716 29255 raft_consensus.cc:2781] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.191495 28721 catalog_manager.cc:5484] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 reported cstate change: term changed from 0 to 1, leader changed from <none> to 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "643a571dd31d4f31b73e1cd53416c561" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: HEALTHY } } }
I20240501 08:10:48.191815 29255 raft_consensus.cc:482] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.191891 29255 raft_consensus.cc:3037] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.192824 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b56cc1f96fd94a15930eef89d48e1e45" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.193018 29123 raft_consensus.cc:2454] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.193866 28796 leader_election.cc:302] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.194089 29194 raft_consensus.cc:2781] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.194188 29194 raft_consensus.cc:482] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.194278 29194 raft_consensus.cc:3037] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.194415 29255 raft_consensus.cc:504] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.194983 29255 leader_election.cc:288] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.196329 29194 raft_consensus.cc:504] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.196483 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c14911bdc3b3444a9a07bc58f7d4ea83" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.196681 28992 raft_consensus.cc:3037] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.196975 29194 leader_election.cc:288] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.197223 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c14911bdc3b3444a9a07bc58f7d4ea83" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.197409 29123 raft_consensus.cc:3037] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.197350 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b56cc1f96fd94a15930eef89d48e1e45" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.197557 28993 raft_consensus.cc:3037] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.199004 28992 raft_consensus.cc:2454] T c14911bdc3b3444a9a07bc58f7d4ea83 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.199671 28795 leader_election.cc:302] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.199916 29194 raft_consensus.cc:2781] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.200114 29123 raft_consensus.cc:2454] T c14911bdc3b3444a9a07bc58f7d4ea83 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.200194 28993 raft_consensus.cc:2454] T b56cc1f96fd94a15930eef89d48e1e45 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.200098 29194 raft_consensus.cc:686] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.200634 28795 leader_election.cc:302] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.200609 29194 consensus_queue.cc:237] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.200836 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b56cc1f96fd94a15930eef89d48e1e45" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.200978 29255 raft_consensus.cc:2781] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.201030 29123 raft_consensus.cc:3037] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.201709 29261 raft_consensus.cc:482] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.201906 29261 raft_consensus.cc:504] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.203163 29122 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.203155 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.203361 28993 raft_consensus.cc:2454] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.203377 29122 raft_consensus.cc:2454] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.203752 29123 raft_consensus.cc:2454] T b56cc1f96fd94a15930eef89d48e1e45 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.203759 28795 leader_election.cc:302] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.204103 29261 leader_election.cc:288] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.204394 29261 raft_consensus.cc:2781] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.204566 29261 raft_consensus.cc:482] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.204725 29261 raft_consensus.cc:3037] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.208509 29261 raft_consensus.cc:504] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.209101 29255 raft_consensus.cc:686] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.209453 29255 consensus_queue.cc:237] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.210191 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.210466 29123 raft_consensus.cc:3037] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.213014 29123 raft_consensus.cc:2454] T 00f202cb3ace4e75983d5e82f6470943 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.213368 29261 leader_election.cc:288] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.213440 28796 leader_election.cc:302] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.213414 29194 raft_consensus.cc:482] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.213744 29261 raft_consensus.cc:2781] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.213781 29194 raft_consensus.cc:504] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.213984 29261 raft_consensus.cc:686] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.214298 29261 consensus_queue.cc:237] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.214757 29194 leader_election.cc:288] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.214996 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af919c096bf845b287bc8eb84ab4c266" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.215183 29123 raft_consensus.cc:2454] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.215587 28796 leader_election.cc:302] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.215922 29194 raft_consensus.cc:2781] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.216095 29194 raft_consensus.cc:482] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.216262 29194 raft_consensus.cc:3037] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.216763 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00f202cb3ace4e75983d5e82f6470943" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.216966 28993 raft_consensus.cc:3037] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.217769 28721 catalog_manager.cc:5484] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.219141 29194 raft_consensus.cc:504] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.219600 28993 raft_consensus.cc:2454] T 00f202cb3ace4e75983d5e82f6470943 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.219986 29194 leader_election.cc:288] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.220234 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af919c096bf845b287bc8eb84ab4c266" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.220409 29123 raft_consensus.cc:3037] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.221249 28993 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af919c096bf845b287bc8eb84ab4c266" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.221383 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af919c096bf845b287bc8eb84ab4c266" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.221446 28993 raft_consensus.cc:2454] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.222584 29123 raft_consensus.cc:2454] T af919c096bf845b287bc8eb84ab4c266 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.222970 28796 leader_election.cc:302] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters: 9c47b857624d401cb096a1b82023b8a6
I20240501 08:10:48.223868 29194 raft_consensus.cc:2781] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.224041 29194 raft_consensus.cc:686] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.224265 29194 consensus_queue.cc:237] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.226554 28721 catalog_manager.cc:5484] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.226985 28721 catalog_manager.cc:5484] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.231613 28721 catalog_manager.cc:5484] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.241212 29194 raft_consensus.cc:482] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.241431 29194 raft_consensus.cc:504] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.242245 29194 leader_election.cc:288] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.244561 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d33c53a8cc2491a8b23a2a05bd9b26d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561" is_pre_election: true
I20240501 08:10:48.244838 29123 raft_consensus.cc:2454] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.244808 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d33c53a8cc2491a8b23a2a05bd9b26d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.245095 28992 raft_consensus.cc:2454] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 0.
I20240501 08:10:48.245575 28795 leader_election.cc:302] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.245908 29194 raft_consensus.cc:2781] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.246042 29194 raft_consensus.cc:482] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.246120 29194 raft_consensus.cc:3037] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.248662 29194 raft_consensus.cc:504] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.249379 29194 leader_election.cc:288] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131:36895)
I20240501 08:10:48.249758 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d33c53a8cc2491a8b23a2a05bd9b26d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.249830 29123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d33c53a8cc2491a8b23a2a05bd9b26d" candidate_uuid: "cd25c86f47a44a01ac0d1a2be410088d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "643a571dd31d4f31b73e1cd53416c561"
I20240501 08:10:48.249998 29123 raft_consensus.cc:3037] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.250003 28992 raft_consensus.cc:3037] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.252727 29123 raft_consensus.cc:2454] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.252714 28992 raft_consensus.cc:2454] T 1d33c53a8cc2491a8b23a2a05bd9b26d P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cd25c86f47a44a01ac0d1a2be410088d in term 1.
I20240501 08:10:48.253121 28795 leader_election.cc:302] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c47b857624d401cb096a1b82023b8a6, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.253448 29194 raft_consensus.cc:2781] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.253598 29194 raft_consensus.cc:686] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [term 1 LEADER]: Becoming Leader. State: Replica: cd25c86f47a44a01ac0d1a2be410088d, State: Running, Role: LEADER
I20240501 08:10:48.254123 29194 consensus_queue.cc:237] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.256978 28721 catalog_manager.cc:5484] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d reported cstate change: term changed from 0 to 1, leader changed from <none> to cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129). New cstate: current_term: 1 leader_uuid: "cd25c86f47a44a01ac0d1a2be410088d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:48.337869 29249 raft_consensus.cc:482] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:48.338097 29249 raft_consensus.cc:504] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.338737 29249 leader_election.cc:288] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033)
I20240501 08:10:48.339349 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "787f5278de3245c9bebf92be0155a5d9" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d" is_pre_election: true
I20240501 08:10:48.339345 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "787f5278de3245c9bebf92be0155a5d9" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6" is_pre_election: true
I20240501 08:10:48.339573 28862 raft_consensus.cc:2454] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.339573 28992 raft_consensus.cc:2454] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 0.
I20240501 08:10:48.339993 29058 leader_election.cc:302] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, cd25c86f47a44a01ac0d1a2be410088d; no voters:
I20240501 08:10:48.340272 29249 raft_consensus.cc:2781] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:48.340385 29249 raft_consensus.cc:482] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:48.340481 29249 raft_consensus.cc:3037] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.342310 29249 raft_consensus.cc:504] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.342913 29249 leader_election.cc:288] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Requested vote from peers 9c47b857624d401cb096a1b82023b8a6 (127.28.1.130:45849), cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033)
I20240501 08:10:48.343284 28992 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "787f5278de3245c9bebf92be0155a5d9" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c47b857624d401cb096a1b82023b8a6"
I20240501 08:10:48.343324 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "787f5278de3245c9bebf92be0155a5d9" candidate_uuid: "643a571dd31d4f31b73e1cd53416c561" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd25c86f47a44a01ac0d1a2be410088d"
I20240501 08:10:48.343464 28992 raft_consensus.cc:3037] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.343482 28862 raft_consensus.cc:3037] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.345542 28992 raft_consensus.cc:2454] T 787f5278de3245c9bebf92be0155a5d9 P 9c47b857624d401cb096a1b82023b8a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.345924 29056 leader_election.cc:302] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 643a571dd31d4f31b73e1cd53416c561, 9c47b857624d401cb096a1b82023b8a6; no voters:
I20240501 08:10:48.346031 28862 raft_consensus.cc:2454] T 787f5278de3245c9bebf92be0155a5d9 P cd25c86f47a44a01ac0d1a2be410088d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 643a571dd31d4f31b73e1cd53416c561 in term 1.
I20240501 08:10:48.346182 29249 raft_consensus.cc:2781] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:48.346369 29249 raft_consensus.cc:686] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [term 1 LEADER]: Becoming Leader. State: Replica: 643a571dd31d4f31b73e1cd53416c561, State: Running, Role: LEADER
I20240501 08:10:48.346623 29249 consensus_queue.cc:237] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } }
I20240501 08:10:48.349063 28721 catalog_manager.cc:5484] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 reported cstate change: term changed from 0 to 1, leader changed from <none> to 643a571dd31d4f31b73e1cd53416c561 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "643a571dd31d4f31b73e1cd53416c561" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 } health_report { overall_health: HEALTHY } } }
I20240501 08:10:48.471931 29192 consensus_queue.cc:1035] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.477705 29192 consensus_queue.cc:1035] T dd52aada8a994e7a9a98aca87a0377b4 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.506309 29192 consensus_queue.cc:1035] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.511188 29239 consensus_queue.cc:1035] T 8724bf810a21401a91ecab6cb9d07588 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.534940 29249 consensus_queue.cc:1035] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.540011 29193 consensus_queue.cc:1035] T 72fb544bcc4643398ce932f884afdfbc P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.570653 29279 consensus_queue.cc:1035] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.576200 29279 consensus_queue.cc:1035] T 309b6208e40e4ea398071b1920f147e1 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.579505 29194 consensus_queue.cc:1035] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.586436 29261 consensus_queue.cc:1035] T b7c75235244d42e1859a7998ff02967d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.588832 29279 consensus_queue.cc:1035] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.603847 29261 consensus_queue.cc:1035] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.614053 29261 consensus_queue.cc:1035] T b56cc1f96fd94a15930eef89d48e1e45 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.615173 29239 consensus_queue.cc:1035] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.617956 29255 consensus_queue.cc:1035] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.618331 29193 consensus_queue.cc:1035] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.625329 29239 consensus_queue.cc:1035] T 9c5cb3ebcb0943018df9dd29ab313c40 P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.626061 29194 consensus_queue.cc:1035] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.631762 29263 consensus_queue.cc:1035] T c14911bdc3b3444a9a07bc58f7d4ea83 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.649528 29261 consensus_queue.cc:1035] T 00f202cb3ace4e75983d5e82f6470943 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.678349 29193 consensus_queue.cc:1035] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.678984 29239 consensus_queue.cc:1035] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.684118 29249 consensus_queue.cc:1035] T 250d9bebc71d4a76957b22897562faca P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.687026 28993 raft_consensus.cc:3037] T af919c096bf845b287bc8eb84ab4c266 P 9c47b857624d401cb096a1b82023b8a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:48.710532 29261 consensus_queue.cc:1035] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.713565 29261 consensus_queue.cc:1035] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.719585 29263 consensus_queue.cc:1035] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.723738 29264 consensus_queue.cc:1035] T 91f59b211cef4cdb87dbc95a78fc412b P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.724303 29239 consensus_queue.cc:1035] T f2cf4e6a8bc346c791ac7b64d823e5bb P 9c47b857624d401cb096a1b82023b8a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.724958 29263 consensus_queue.cc:1035] T 1d33c53a8cc2491a8b23a2a05bd9b26d P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.731392 29264 consensus_queue.cc:1035] T af919c096bf845b287bc8eb84ab4c266 P cd25c86f47a44a01ac0d1a2be410088d [LEADER]: Connected to new peer: Peer: permanent_uuid: "643a571dd31d4f31b73e1cd53416c561" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 36895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:10:48.789237 29270 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:48.789546 29270 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:48.797202 29270 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:10:48.825560 29279 consensus_queue.cc:1035] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd25c86f47a44a01ac0d1a2be410088d" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 43033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:48.830750 29249 consensus_queue.cc:1035] T 787f5278de3245c9bebf92be0155a5d9 P 643a571dd31d4f31b73e1cd53416c561 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c47b857624d401cb096a1b82023b8a6" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45849 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 314.386 ms
time per row: 0.157193 ms
I20240501 08:10:49.721710 28973 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:10:49.722326 28837 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240501 08:10:49.724655 29102 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:10:49.900828 29102 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:10:49.909683 28837 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240501 08:10:49.917680 28973 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:10:50.096038 29102 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:10:50.097295 28973 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:10:50.099049 28837 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240501 08:10:50.722045 29102 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:10:50.727519 28973 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:10:50.740834 28837 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240501 08:10:50.895861 28973 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:10:50.896366 28837 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240501 08:10:50.897061 29102 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:10:51.042441 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 28780
I20240501 08:10:51.048629 28902 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:51.168802 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 28780
W20240501 08:10:51.184805 29059 connection.cc:535] server connection from 127.28.1.129:59559 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:10:51.185302 29058 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.28.1.129:43033 (error 108)
I20240501 08:10:51.185585 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 28911
W20240501 08:10:51.187604 29058 consensus_peers.cc:485] T 49d6e21f117b4738aafa3d7ebaffabdb P 643a571dd31d4f31b73e1cd53416c561 -> Peer cd25c86f47a44a01ac0d1a2be410088d (127.28.1.129:43033): Couldn't send request to peer cd25c86f47a44a01ac0d1a2be410088d. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:43033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:10:51.191076 29033 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:51.310694 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 28911
I20240501 08:10:51.326664 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 29041
I20240501 08:10:51.331941 29163 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:51.451192 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 29041
I20240501 08:10:51.468413 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 28690
I20240501 08:10:51.469578 28751 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:51.594359 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 28690
2024-05-01T08:10:51Z chronyd exiting
[ OK ] TableRangeRebalancingTest.Basic (9690 ms)
[----------] 1 test from TableRangeRebalancingTest (9690 ms total)
[----------] 1 test from RebalanceStartSafetyTest
[ RUN ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1
I20240501 08:10:51.640568 28678 test_util.cc:268] Using random seed: 107077340
I20240501 08:10:51.642370 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:10:51.642448 28678 ts_itest-base.cc:113] --------------
I20240501 08:10:51.642506 28678 ts_itest-base.cc:114] 5 tablet servers
I20240501 08:10:51.642550 28678 ts_itest-base.cc:115] 3 replicas per TS
I20240501 08:10:51.642611 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:10:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:10:51Z Disabled control of system clock
I20240501 08:10:51.663769 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:45371
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:45371
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240501 08:10:51.994493 29537 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:51.994834 29537 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:51.994962 29537 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:52.002470 29537 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:52.002610 29537 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:10:52.002679 29537 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:52.002758 29537 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:10:52.002832 29537 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:10:52.014314 29537 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:45371
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:45371
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:52.016296 29537 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:52.018036 29537 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:52.028571 29545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:52.029052 29542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:52.029670 29543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:52.030313 29537 server_base.cc:1034] running on GCE node
I20240501 08:10:52.031131 29537 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:52.032547 29537 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:52.033753 29537 hybrid_clock.cc:648] HybridClock initialized: now 1714551052033706 us; error 39 us; skew 500 ppm
I20240501 08:10:52.034092 29537 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:52.036336 29537 webserver.cc:460] Webserver started at http://127.28.1.190:42009/ using document root <none> and password file <none>
I20240501 08:10:52.037029 29537 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:52.037138 29537 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:52.037456 29537 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:52.040012 29537 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "1a19ca31d08c493282f8d43329bf35fe"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.040761 29537 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "1a19ca31d08c493282f8d43329bf35fe"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.045909 29537 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:52.049238 29551 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:52.050596 29537 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:10:52.050763 29537 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "1a19ca31d08c493282f8d43329bf35fe"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.050980 29537 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:52.094782 29537 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:52.095674 29537 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:52.095986 29537 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:52.115070 29602 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:45371 every 8 connection(s)
I20240501 08:10:52.115111 29537 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:45371
I20240501 08:10:52.116752 29537 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:10:52.119426 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 29537
I20240501 08:10:52.119626 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:10:52.120972 29603 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:52.133335 29603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe: Bootstrap starting.
I20240501 08:10:52.137838 29603 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:52.139147 29603 log.cc:824] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:52.141944 29603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe: No bootstrap required, opened a new log
I20240501 08:10:52.147955 29603 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } }
I20240501 08:10:52.148346 29603 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:52.148447 29603 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a19ca31d08c493282f8d43329bf35fe, State: Initialized, Role: FOLLOWER
I20240501 08:10:52.149014 29603 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } }
I20240501 08:10:52.149195 29603 raft_consensus.cc:388] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:10:52.149313 29603 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:10:52.149448 29603 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:52.210224 29603 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } }
I20240501 08:10:52.211068 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 1a19ca31d08c493282f8d43329bf35fe; no voters:
I20240501 08:10:52.211710 29603 leader_election.cc:288] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:10:52.211880 29608 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:52.212980 29608 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [term 1 LEADER]: Becoming Leader. State: Replica: 1a19ca31d08c493282f8d43329bf35fe, State: Running, Role: LEADER
I20240501 08:10:52.213622 29608 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } }
I20240501 08:10:52.213938 29603 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:10:52.216635 29610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a19ca31d08c493282f8d43329bf35fe. Latest consensus state: current_term: 1 leader_uuid: "1a19ca31d08c493282f8d43329bf35fe" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } } }
I20240501 08:10:52.217067 29610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:52.216681 29609 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "1a19ca31d08c493282f8d43329bf35fe" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1a19ca31d08c493282f8d43329bf35fe" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45371 } } }
I20240501 08:10:52.217355 29609 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:52.219053 29616 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:10:52.223820 29616 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:10:52.231644 29616 catalog_manager.cc:1347] Generated new cluster ID: 4fb62be811e94dbe86dac72ac3937915
I20240501 08:10:52.231765 29616 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:10:52.260751 29616 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:10:52.262601 29616 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:10:52.283303 29616 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 1a19ca31d08c493282f8d43329bf35fe: Generated new TSK 0
I20240501 08:10:52.284518 29616 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:10:52.304713 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:10:52.618449 29627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:52.618768 29627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:52.618914 29627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:52.626220 29627 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:52.626333 29627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:52.626482 29627 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:10:52.635741 29627 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:52.637068 29627 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:52.638926 29627 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:52.650521 29632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:52.651141 29633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:52.651501 29635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:52.652118 29627 server_base.cc:1034] running on GCE node
I20240501 08:10:52.652663 29627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:52.653571 29627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:52.654753 29627 hybrid_clock.cc:648] HybridClock initialized: now 1714551052654699 us; error 53 us; skew 500 ppm
I20240501 08:10:52.655046 29627 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:52.657223 29627 webserver.cc:460] Webserver started at http://127.28.1.129:42223/ using document root <none> and password file <none>
I20240501 08:10:52.657908 29627 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:52.658013 29627 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:52.658316 29627 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:52.660831 29627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "21eaf98a8dd84c249e2feadbf70cf0b6"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.661540 29627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "21eaf98a8dd84c249e2feadbf70cf0b6"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.667183 29627 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:52.670327 29641 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:52.671540 29627 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240501 08:10:52.671674 29627 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "21eaf98a8dd84c249e2feadbf70cf0b6"
format_stamp: "Formatted at 2024-05-01 08:10:52 on dist-test-slave-jj4n"
I20240501 08:10:52.671854 29627 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:52.697611 29627 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:52.698469 29627 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:52.698753 29627 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:52.699738 29627 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:52.701256 29627 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:52.701350 29627 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:52.701459 29627 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:52.701534 29627 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:52.731060 29627 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:42269
I20240501 08:10:52.731127 29753 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:42269 every 8 connection(s)
I20240501 08:10:52.732614 29627 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:10:52.740268 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 29627
I20240501 08:10:52.740468 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:10:52.743839 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:52.745298 29754 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45371
I20240501 08:10:52.745586 29754 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:52.746361 29754 heartbeater.cc:511] Master 127.28.1.190:45371 requested a full tablet report, sending...
I20240501 08:10:52.748508 29568 ts_manager.cc:194] Registered new tserver with Master: 21eaf98a8dd84c249e2feadbf70cf0b6 (127.28.1.129:42269)
I20240501 08:10:52.750079 29568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:46811
W20240501 08:10:53.055581 29758 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:53.055938 29758 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:53.056105 29758 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:53.063418 29758 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:53.063572 29758 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:53.063740 29758 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:10:53.072818 29758 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:53.074172 29758 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:53.075915 29758 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:53.087666 29763 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.088140 29766 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.088536 29764 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:53.089160 29758 server_base.cc:1034] running on GCE node
I20240501 08:10:53.089752 29758 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:53.090709 29758 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:53.091925 29758 hybrid_clock.cc:648] HybridClock initialized: now 1714551053091869 us; error 42 us; skew 500 ppm
I20240501 08:10:53.092283 29758 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:53.094553 29758 webserver.cc:460] Webserver started at http://127.28.1.130:46841/ using document root <none> and password file <none>
I20240501 08:10:53.095278 29758 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:53.095389 29758 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:53.095692 29758 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:53.098251 29758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "55e2239b65c3471ba25ca6fc8383f933"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.098965 29758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "55e2239b65c3471ba25ca6fc8383f933"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.104125 29758 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
I20240501 08:10:53.107260 29772 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.108521 29758 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:10:53.108659 29758 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "55e2239b65c3471ba25ca6fc8383f933"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.108829 29758 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:53.129045 29758 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:53.129889 29758 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:53.130151 29758 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:53.131091 29758 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:53.132545 29758 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:53.132638 29758 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.132743 29758 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:53.132809 29758 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.161451 29758 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:44629
I20240501 08:10:53.161504 29884 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:44629 every 8 connection(s)
I20240501 08:10:53.162676 29758 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:10:53.170578 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 29758
I20240501 08:10:53.170755 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:10:53.174041 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:53.175931 29885 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45371
I20240501 08:10:53.176301 29885 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:53.177085 29885 heartbeater.cc:511] Master 127.28.1.190:45371 requested a full tablet report, sending...
I20240501 08:10:53.178401 29568 ts_manager.cc:194] Registered new tserver with Master: 55e2239b65c3471ba25ca6fc8383f933 (127.28.1.130:44629)
I20240501 08:10:53.179145 29568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:53189
W20240501 08:10:53.480129 29888 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:53.480403 29888 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:53.480540 29888 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:53.488010 29888 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:53.488168 29888 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:53.488361 29888 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:10:53.497491 29888 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:53.498767 29888 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:53.500509 29888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:53.513020 29896 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.513868 29893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.514331 29894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:53.514945 29888 server_base.cc:1034] running on GCE node
I20240501 08:10:53.515620 29888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:53.516672 29888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:53.517887 29888 hybrid_clock.cc:648] HybridClock initialized: now 1714551053517835 us; error 48 us; skew 500 ppm
I20240501 08:10:53.518216 29888 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:53.520539 29888 webserver.cc:460] Webserver started at http://127.28.1.131:46349/ using document root <none> and password file <none>
I20240501 08:10:53.521291 29888 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:53.521418 29888 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:53.521746 29888 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:53.524417 29888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "f5bc3aeea6414c61a0b3915953ae1d41"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.525171 29888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "f5bc3aeea6414c61a0b3915953ae1d41"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.530207 29888 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
I20240501 08:10:53.533494 29902 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.534806 29888 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240501 08:10:53.534955 29888 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "f5bc3aeea6414c61a0b3915953ae1d41"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.535146 29888 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:53.561205 29888 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:53.562089 29888 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:53.562376 29888 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:53.563357 29888 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:53.564957 29888 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:53.565055 29888 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.565168 29888 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:53.565236 29888 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.595250 29888 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:36703
I20240501 08:10:53.595295 30014 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:36703 every 8 connection(s)
I20240501 08:10:53.596622 29888 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:10:53.599871 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 29888
I20240501 08:10:53.600078 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:10:53.603281 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:53.610110 30015 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45371
I20240501 08:10:53.610420 30015 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:53.611310 30015 heartbeater.cc:511] Master 127.28.1.190:45371 requested a full tablet report, sending...
I20240501 08:10:53.612762 29568 ts_manager.cc:194] Registered new tserver with Master: f5bc3aeea6414c61a0b3915953ae1d41 (127.28.1.131:36703)
I20240501 08:10:53.613549 29568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:35491
I20240501 08:10:53.752841 29754 heartbeater.cc:503] Master 127.28.1.190:45371 was elected leader, sending a full tablet report...
W20240501 08:10:53.919734 30018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:53.920085 30018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:53.920233 30018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:53.927659 30018 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:53.927801 30018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:53.927995 30018 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:10:53.937153 30018 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:53.938584 30018 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:53.940505 30018 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:53.952589 30023 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.953135 30024 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:53.953574 30026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:53.954110 30018 server_base.cc:1034] running on GCE node
I20240501 08:10:53.954749 30018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:53.955731 30018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:53.956918 30018 hybrid_clock.cc:648] HybridClock initialized: now 1714551053956834 us; error 72 us; skew 500 ppm
I20240501 08:10:53.957350 30018 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:53.967533 30018 webserver.cc:460] Webserver started at http://127.28.1.132:33631/ using document root <none> and password file <none>
I20240501 08:10:53.968366 30018 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:53.968477 30018 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:53.968804 30018 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:53.971477 30018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "b333a8c8ff7d4519b0be3c70d2eda508"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.972296 30018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "b333a8c8ff7d4519b0be3c70d2eda508"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.977589 30018 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.001s sys 0.003s
I20240501 08:10:53.981025 30032 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:53.982442 30018 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240501 08:10:53.982599 30018 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "b333a8c8ff7d4519b0be3c70d2eda508"
format_stamp: "Formatted at 2024-05-01 08:10:53 on dist-test-slave-jj4n"
I20240501 08:10:53.982787 30018 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:54.009253 30018 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:54.010133 30018 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:54.010413 30018 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:54.011358 30018 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:54.012966 30018 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:54.013062 30018 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:54.013175 30018 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:54.013240 30018 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:54.042304 30018 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:35321
I20240501 08:10:54.042374 30144 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:35321 every 8 connection(s)
I20240501 08:10:54.043591 30018 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:10:54.051514 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30018
I20240501 08:10:54.051681 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:10:54.054977 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.133:0
--local_ip_for_outbound_sockets=127.28.1.133
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--builtin_ntp_servers=127.28.1.148:41011
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:54.056372 30145 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45371
I20240501 08:10:54.056667 30145 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:54.057461 30145 heartbeater.cc:511] Master 127.28.1.190:45371 requested a full tablet report, sending...
I20240501 08:10:54.058789 29568 ts_manager.cc:194] Registered new tserver with Master: b333a8c8ff7d4519b0be3c70d2eda508 (127.28.1.132:35321)
I20240501 08:10:54.059679 29568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:41341
I20240501 08:10:54.181560 29885 heartbeater.cc:503] Master 127.28.1.190:45371 was elected leader, sending a full tablet report...
W20240501 08:10:54.381203 30148 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:54.381507 30148 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:54.381608 30148 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:54.389140 30148 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:54.389272 30148 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:54.389431 30148 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.133
I20240501 08:10:54.398572 30148 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41011
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45371
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.133
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:54.399857 30148 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:54.401566 30148 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:54.413224 30156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:54.413483 30154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:54.413916 30153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:54.414635 30148 server_base.cc:1034] running on GCE node
I20240501 08:10:54.415279 30148 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:54.416235 30148 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:54.417433 30148 hybrid_clock.cc:648] HybridClock initialized: now 1714551054417382 us; error 39 us; skew 500 ppm
I20240501 08:10:54.417719 30148 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:54.419926 30148 webserver.cc:460] Webserver started at http://127.28.1.133:45161/ using document root <none> and password file <none>
I20240501 08:10:54.420650 30148 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:54.420743 30148 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:54.421011 30148 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:54.423477 30148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "1d860adda5b148afa35b1c46a95dc767"
format_stamp: "Formatted at 2024-05-01 08:10:54 on dist-test-slave-jj4n"
I20240501 08:10:54.424223 30148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "1d860adda5b148afa35b1c46a95dc767"
format_stamp: "Formatted at 2024-05-01 08:10:54 on dist-test-slave-jj4n"
I20240501 08:10:54.429131 30148 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240501 08:10:54.432399 30162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:54.433671 30148 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
I20240501 08:10:54.433817 30148 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "1d860adda5b148afa35b1c46a95dc767"
format_stamp: "Formatted at 2024-05-01 08:10:54 on dist-test-slave-jj4n"
I20240501 08:10:54.434000 30148 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:54.465155 30148 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:54.466042 30148 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:54.466295 30148 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:54.467231 30148 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:54.468688 30148 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:54.468772 30148 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:54.468860 30148 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:54.468914 30148 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:54.497562 30148 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.133:45347
I20240501 08:10:54.497618 30274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.133:45347 every 8 connection(s)
I20240501 08:10:54.498847 30148 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240501 08:10:54.506242 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30148
I20240501 08:10:54.506418 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240501 08:10:54.511005 30275 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45371
I20240501 08:10:54.511272 30275 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:54.512069 30275 heartbeater.cc:511] Master 127.28.1.190:45371 requested a full tablet report, sending...
I20240501 08:10:54.513406 29568 ts_manager.cc:194] Registered new tserver with Master: 1d860adda5b148afa35b1c46a95dc767 (127.28.1.133:45347)
I20240501 08:10:54.514194 29568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.133:36677
I20240501 08:10:54.521790 28678 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240501 08:10:54.536499 29568 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46622:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240501 08:10:54.576531 30210 tablet_service.cc:1467] Processing CreateTablet for tablet 6054a0f5e149474f87f5a5e923cc1c24 (DEFAULT_TABLE table=TestTable [id=01adb9089cc7478dbda2736471e833c4]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:10:54.576702 29820 tablet_service.cc:1467] Processing CreateTablet for tablet 6054a0f5e149474f87f5a5e923cc1c24 (DEFAULT_TABLE table=TestTable [id=01adb9089cc7478dbda2736471e833c4]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:10:54.577289 29689 tablet_service.cc:1467] Processing CreateTablet for tablet 6054a0f5e149474f87f5a5e923cc1c24 (DEFAULT_TABLE table=TestTable [id=01adb9089cc7478dbda2736471e833c4]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:10:54.577952 30210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6054a0f5e149474f87f5a5e923cc1c24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:54.578605 29689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6054a0f5e149474f87f5a5e923cc1c24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:54.578625 29820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6054a0f5e149474f87f5a5e923cc1c24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:54.588253 30293 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Bootstrap starting.
I20240501 08:10:54.591529 30293 tablet_bootstrap.cc:654] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:54.591862 30294 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Bootstrap starting.
I20240501 08:10:54.593133 30293 log.cc:824] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:54.593967 30295 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Bootstrap starting.
I20240501 08:10:54.596838 30293 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: No bootstrap required, opened a new log
I20240501 08:10:54.597124 30293 ts_tablet_manager.cc:1367] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20240501 08:10:54.597911 30294 tablet_bootstrap.cc:654] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:54.598896 30294 log.cc:824] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:54.602715 30295 tablet_bootstrap.cc:654] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:54.602962 30294 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: No bootstrap required, opened a new log
I20240501 08:10:54.603231 30294 ts_tablet_manager.cc:1367] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Time spent bootstrapping tablet: real 0.012s user 0.005s sys 0.000s
I20240501 08:10:54.603695 30295 log.cc:824] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:54.605321 30293 raft_consensus.cc:348] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.605765 30293 raft_consensus.cc:374] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:54.605911 30293 raft_consensus.cc:729] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d860adda5b148afa35b1c46a95dc767, State: Initialized, Role: FOLLOWER
I20240501 08:10:54.606041 30295 tablet_bootstrap.cc:492] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: No bootstrap required, opened a new log
I20240501 08:10:54.606317 30295 ts_tablet_manager.cc:1367] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.000s
I20240501 08:10:54.606623 30293 consensus_queue.cc:260] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.607717 30275 heartbeater.cc:503] Master 127.28.1.190:45371 was elected leader, sending a full tablet report...
I20240501 08:10:54.609854 30293 ts_tablet_manager.cc:1398] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Time spent starting tablet: real 0.013s user 0.011s sys 0.000s
I20240501 08:10:54.611755 30294 raft_consensus.cc:348] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.612203 30294 raft_consensus.cc:374] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:54.612316 30294 raft_consensus.cc:729] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 55e2239b65c3471ba25ca6fc8383f933, State: Initialized, Role: FOLLOWER
I20240501 08:10:54.613015 30294 consensus_queue.cc:260] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.614038 30295 raft_consensus.cc:348] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.614539 30295 raft_consensus.cc:374] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:54.614675 30295 raft_consensus.cc:729] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 21eaf98a8dd84c249e2feadbf70cf0b6, State: Initialized, Role: FOLLOWER
I20240501 08:10:54.614806 30294 ts_tablet_manager.cc:1398] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
I20240501 08:10:54.615306 30295 consensus_queue.cc:260] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.617002 30295 ts_tablet_manager.cc:1398] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Time spent starting tablet: real 0.011s user 0.007s sys 0.004s
I20240501 08:10:54.617488 30015 heartbeater.cc:503] Master 127.28.1.190:45371 was elected leader, sending a full tablet report...
I20240501 08:10:54.648003 30299 raft_consensus.cc:482] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:10:54.648267 30299 raft_consensus.cc:504] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.650489 30299 leader_election.cc:288] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 21eaf98a8dd84c249e2feadbf70cf0b6 (127.28.1.129:42269), 55e2239b65c3471ba25ca6fc8383f933 (127.28.1.130:44629)
I20240501 08:10:54.656728 29840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6054a0f5e149474f87f5a5e923cc1c24" candidate_uuid: "1d860adda5b148afa35b1c46a95dc767" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "55e2239b65c3471ba25ca6fc8383f933" is_pre_election: true
I20240501 08:10:54.656728 29709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6054a0f5e149474f87f5a5e923cc1c24" candidate_uuid: "1d860adda5b148afa35b1c46a95dc767" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" is_pre_election: true
I20240501 08:10:54.657272 29840 raft_consensus.cc:2454] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1d860adda5b148afa35b1c46a95dc767 in term 0.
I20240501 08:10:54.657272 29709 raft_consensus.cc:2454] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1d860adda5b148afa35b1c46a95dc767 in term 0.
I20240501 08:10:54.657938 30166 leader_election.cc:302] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d860adda5b148afa35b1c46a95dc767, 21eaf98a8dd84c249e2feadbf70cf0b6; no voters:
I20240501 08:10:54.658334 30299 raft_consensus.cc:2781] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:10:54.658464 30299 raft_consensus.cc:482] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:10:54.658550 30299 raft_consensus.cc:3037] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 0 FOLLOWER]: Advancing to term 1
W20240501 08:10:54.665277 29886 tablet.cc:2369] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:10:54.736411 29755 tablet.cc:2369] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:10:54.750442 30276 tablet.cc:2369] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:10:54.751907 30299 raft_consensus.cc:504] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.752544 30299 leader_election.cc:288] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [CANDIDATE]: Term 1 election: Requested vote from peers 21eaf98a8dd84c249e2feadbf70cf0b6 (127.28.1.129:42269), 55e2239b65c3471ba25ca6fc8383f933 (127.28.1.130:44629)
I20240501 08:10:54.753079 29709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6054a0f5e149474f87f5a5e923cc1c24" candidate_uuid: "1d860adda5b148afa35b1c46a95dc767" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6"
I20240501 08:10:54.753145 29840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6054a0f5e149474f87f5a5e923cc1c24" candidate_uuid: "1d860adda5b148afa35b1c46a95dc767" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "55e2239b65c3471ba25ca6fc8383f933"
I20240501 08:10:54.753271 29709 raft_consensus.cc:3037] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:54.753363 29840 raft_consensus.cc:3037] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:54.755422 29709 raft_consensus.cc:2454] T 6054a0f5e149474f87f5a5e923cc1c24 P 21eaf98a8dd84c249e2feadbf70cf0b6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1d860adda5b148afa35b1c46a95dc767 in term 1.
I20240501 08:10:54.755838 30166 leader_election.cc:302] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d860adda5b148afa35b1c46a95dc767, 21eaf98a8dd84c249e2feadbf70cf0b6; no voters:
I20240501 08:10:54.756161 30299 raft_consensus.cc:2781] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:54.756587 30299 raft_consensus.cc:686] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [term 1 LEADER]: Becoming Leader. State: Replica: 1d860adda5b148afa35b1c46a95dc767, State: Running, Role: LEADER
I20240501 08:10:54.757038 30299 consensus_queue.cc:237] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } }
I20240501 08:10:54.761780 29567 catalog_manager.cc:5484] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1d860adda5b148afa35b1c46a95dc767 (127.28.1.133). New cstate: current_term: 1 leader_uuid: "1d860adda5b148afa35b1c46a95dc767" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1d860adda5b148afa35b1c46a95dc767" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 45347 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 } health_report { overall_health: UNKNOWN } } }
I20240501 08:10:54.803696 28678 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240501 08:10:54.806190 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 21eaf98a8dd84c249e2feadbf70cf0b6 to finish bootstrapping
I20240501 08:10:54.814944 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 55e2239b65c3471ba25ca6fc8383f933 to finish bootstrapping
I20240501 08:10:54.849452 29840 raft_consensus.cc:2454] T 6054a0f5e149474f87f5a5e923cc1c24 P 55e2239b65c3471ba25ca6fc8383f933 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1d860adda5b148afa35b1c46a95dc767 in term 1.
I20240501 08:10:54.851593 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver f5bc3aeea6414c61a0b3915953ae1d41 to finish bootstrapping
I20240501 08:10:54.860252 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver b333a8c8ff7d4519b0be3c70d2eda508 to finish bootstrapping
I20240501 08:10:54.868631 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 1d860adda5b148afa35b1c46a95dc767 to finish bootstrapping
I20240501 08:10:55.062325 30145 heartbeater.cc:503] Master 127.28.1.190:45371 was elected leader, sending a full tablet report...
I20240501 08:10:55.157936 30299 consensus_queue.cc:1035] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21eaf98a8dd84c249e2feadbf70cf0b6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42269 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:55.178331 30313 consensus_queue.cc:1035] T 6054a0f5e149474f87f5a5e923cc1c24 P 1d860adda5b148afa35b1c46a95dc767 [LEADER]: Connected to new peer: Peer: permanent_uuid: "55e2239b65c3471ba25ca6fc8383f933" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 44629 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:10:55.336668 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:55.343786 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.351300 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.356853 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.382464 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.521401 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.524642 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:55.533838 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.546023 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.556200 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.711822 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:55.712560 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.719928 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.734702 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:55.747431 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.344781 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:56.344770 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.346868 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.351424 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.376153 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.516436 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:56.518699 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.519155 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.522886 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.534790 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.702929 30210 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:10:56.704746 29689 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.706521 29820 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.706705 30080 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.726547 29950 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:10:56.873430 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 29627
I20240501 08:10:56.874778 29749 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:56.968791 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 29627
I20240501 08:10:56.983651 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 29758
I20240501 08:10:56.984642 29879 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:57.078534 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 29758
I20240501 08:10:57.092110 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 29888
I20240501 08:10:57.093061 30009 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:57.181967 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 29888
I20240501 08:10:57.195433 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30018
I20240501 08:10:57.196410 30139 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:57.285859 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30018
I20240501 08:10:57.299189 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30148
I20240501 08:10:57.300213 30270 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:57.395982 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30148
I20240501 08:10:57.409719 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 29537
I20240501 08:10:57.410746 29598 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:10:57.522624 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 29537
2024-05-01T08:10:57Z chronyd exiting
[ OK ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1 (5918 ms)
[----------] 1 test from RebalanceStartSafetyTest (5918 ms total)
[----------] 1 test from RebalanceParamTest
[ RUN ] RebalanceParamTest.Rebalance/7
I20240501 08:10:57.559438 28678 test_util.cc:268] Using random seed: 112996198
I20240501 08:10:57.561276 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:10:57.561363 28678 ts_itest-base.cc:113] --------------
I20240501 08:10:57.561404 28678 ts_itest-base.cc:114] 7 tablet servers
I20240501 08:10:57.561455 28678 ts_itest-base.cc:115] 5 replicas per TS
I20240501 08:10:57.561491 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:10:57Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:10:57Z Disabled control of system clock
I20240501 08:10:57.582896 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:35983
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:35983
--allow_unsafe_replication_factor
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000 with env {}
W20240501 08:10:57.893905 30537 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240501 08:10:57.894240 30537 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:57.894337 30537 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:57.894431 30537 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:57.901775 30537 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:57.901902 30537 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:10:57.901993 30537 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:57.902047 30537 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:10:57.902129 30537 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:10:57.911198 30537 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:35983
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:35983
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:57.912561 30537 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:57.914279 30537 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:57.924743 30542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:57.925376 30545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:57.925923 30543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:57.926443 30537 server_base.cc:1034] running on GCE node
I20240501 08:10:57.927225 30537 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:57.928663 30537 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:57.929894 30537 hybrid_clock.cc:648] HybridClock initialized: now 1714551057929829 us; error 45 us; skew 500 ppm
I20240501 08:10:57.930224 30537 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:57.932451 30537 webserver.cc:460] Webserver started at http://127.28.1.190:35517/ using document root <none> and password file <none>
I20240501 08:10:57.933146 30537 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:57.933269 30537 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:57.933553 30537 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:57.936268 30537 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "b5013b5f37d84aa1810fafbf49ae9861"
format_stamp: "Formatted at 2024-05-01 08:10:57 on dist-test-slave-jj4n"
I20240501 08:10:57.936985 30537 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "b5013b5f37d84aa1810fafbf49ae9861"
format_stamp: "Formatted at 2024-05-01 08:10:57 on dist-test-slave-jj4n"
I20240501 08:10:57.942111 30537 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:57.945453 30551 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:57.946954 30537 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240501 08:10:57.947163 30537 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "b5013b5f37d84aa1810fafbf49ae9861"
format_stamp: "Formatted at 2024-05-01 08:10:57 on dist-test-slave-jj4n"
I20240501 08:10:57.947355 30537 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:57.972671 30537 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:57.973531 30537 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:57.973822 30537 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:57.990550 30537 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:35983
I20240501 08:10:57.990571 30602 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:35983 every 8 connection(s)
I20240501 08:10:57.991909 30537 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:10:57.995553 30603 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:10:57.997047 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30537
I20240501 08:10:57.997238 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:10:58.005563 30603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861: Bootstrap starting.
I20240501 08:10:58.008683 30603 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861: Neither blocks nor log segments found. Creating new log.
I20240501 08:10:58.009968 30603 log.cc:824] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861: Log is configured to *not* fsync() on all Append() calls
I20240501 08:10:58.012616 30603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861: No bootstrap required, opened a new log
I20240501 08:10:58.018746 30603 raft_consensus.cc:348] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } }
I20240501 08:10:58.019079 30603 raft_consensus.cc:374] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:10:58.019181 30603 raft_consensus.cc:729] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5013b5f37d84aa1810fafbf49ae9861, State: Initialized, Role: FOLLOWER
I20240501 08:10:58.019784 30603 consensus_queue.cc:260] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } }
I20240501 08:10:58.020030 30603 raft_consensus.cc:388] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:10:58.020146 30603 raft_consensus.cc:482] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:10:58.020293 30603 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:10:58.022322 30603 raft_consensus.cc:504] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } }
I20240501 08:10:58.022801 30603 leader_election.cc:302] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b5013b5f37d84aa1810fafbf49ae9861; no voters:
I20240501 08:10:58.023423 30603 leader_election.cc:288] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:10:58.023568 30608 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:10:58.024216 30608 raft_consensus.cc:686] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [term 1 LEADER]: Becoming Leader. State: Replica: b5013b5f37d84aa1810fafbf49ae9861, State: Running, Role: LEADER
I20240501 08:10:58.024696 30608 consensus_queue.cc:237] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } }
I20240501 08:10:58.025403 30603 sys_catalog.cc:564] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:10:58.027213 30610 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5013b5f37d84aa1810fafbf49ae9861. Latest consensus state: current_term: 1 leader_uuid: "b5013b5f37d84aa1810fafbf49ae9861" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } } }
I20240501 08:10:58.027189 30609 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "b5013b5f37d84aa1810fafbf49ae9861" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b5013b5f37d84aa1810fafbf49ae9861" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 35983 } } }
I20240501 08:10:58.027626 30609 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:58.027623 30610 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861 [sys.catalog]: This master's current role is: LEADER
I20240501 08:10:58.029256 30613 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:10:58.034266 30613 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:10:58.042843 30613 catalog_manager.cc:1347] Generated new cluster ID: 4212cc31dcd64e769309f862b2dbccad
I20240501 08:10:58.042969 30613 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:10:58.076524 30613 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:10:58.077718 30613 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:10:58.084884 30613 catalog_manager.cc:5857] T 00000000000000000000000000000000 P b5013b5f37d84aa1810fafbf49ae9861: Generated new TSK 0
I20240501 08:10:58.085791 30613 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:10:58.104147 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:10:58.454594 30627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:58.454880 30627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:58.455014 30627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:58.462481 30627 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:58.462608 30627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:58.462761 30627 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:10:58.471601 30627 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:58.472900 30627 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:58.474493 30627 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:58.485780 30632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:58.486313 30633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:58.486866 30635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:58.487401 30627 server_base.cc:1034] running on GCE node
I20240501 08:10:58.487988 30627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:58.488884 30627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:58.490078 30627 hybrid_clock.cc:648] HybridClock initialized: now 1714551058490024 us; error 48 us; skew 500 ppm
I20240501 08:10:58.490388 30627 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:58.492518 30627 webserver.cc:460] Webserver started at http://127.28.1.129:39293/ using document root <none> and password file <none>
I20240501 08:10:58.493228 30627 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:58.493330 30627 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:58.493628 30627 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:58.496140 30627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.496860 30627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.501726 30627 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:10:58.504962 30641 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.506297 30627 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240501 08:10:58.506453 30627 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.506646 30627 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:58.521773 30627 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:58.522584 30627 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:58.522837 30627 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:58.523727 30627 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:58.525218 30627 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:58.525303 30627 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.525400 30627 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:58.525465 30627 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.552860 30627 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:39637
I20240501 08:10:58.552928 30753 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:39637 every 8 connection(s)
I20240501 08:10:58.554126 30627 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:10:58.562727 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30627
I20240501 08:10:58.562944 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:10:58.566330 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:58.566617 30754 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:10:58.566895 30754 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:58.567732 30754 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:10:58.572250 30568 ts_manager.cc:194] Registered new tserver with Master: 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637)
I20240501 08:10:58.574333 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:35485
W20240501 08:10:58.879374 30758 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:58.879660 30758 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:58.879755 30758 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:58.887207 30758 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:58.887331 30758 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:58.887452 30758 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:10:58.896436 30758 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:58.897660 30758 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:58.899284 30758 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:58.911070 30763 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:58.911501 30766 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:58.912127 30764 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:58.912802 30758 server_base.cc:1034] running on GCE node
I20240501 08:10:58.913405 30758 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:58.914358 30758 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:58.915561 30758 hybrid_clock.cc:648] HybridClock initialized: now 1714551058915512 us; error 41 us; skew 500 ppm
I20240501 08:10:58.915886 30758 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:58.918035 30758 webserver.cc:460] Webserver started at http://127.28.1.130:34505/ using document root <none> and password file <none>
I20240501 08:10:58.918761 30758 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:58.918864 30758 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:58.919163 30758 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:58.921739 30758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "1a388864321846929919a37b570e8138"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.922536 30758 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "1a388864321846929919a37b570e8138"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.927594 30758 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240501 08:10:58.930896 30772 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.932380 30758 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:10:58.932515 30758 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "1a388864321846929919a37b570e8138"
format_stamp: "Formatted at 2024-05-01 08:10:58 on dist-test-slave-jj4n"
I20240501 08:10:58.932688 30758 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:58.951334 30758 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:58.952255 30758 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:58.952508 30758 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:58.953451 30758 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:58.954952 30758 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:58.955035 30758 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.955132 30758 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:58.955188 30758 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:58.984129 30758 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:46291
I20240501 08:10:58.984184 30884 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:46291 every 8 connection(s)
I20240501 08:10:58.985420 30758 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:10:58.994050 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30758
I20240501 08:10:58.994252 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:10:58.998126 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:59.006695 30885 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:10:59.007027 30885 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:59.007910 30885 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:10:59.009526 30568 ts_manager.cc:194] Registered new tserver with Master: 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:10:59.010301 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:38799
W20240501 08:10:59.314504 30888 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:59.314807 30888 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:59.314925 30888 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:59.322288 30888 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:59.322428 30888 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:59.322588 30888 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:10:59.331686 30888 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:59.333009 30888 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:59.334738 30888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:59.346614 30894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:59.347103 30893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:59.347689 30896 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:59.348351 30888 server_base.cc:1034] running on GCE node
I20240501 08:10:59.348937 30888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:59.349866 30888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:59.351058 30888 hybrid_clock.cc:648] HybridClock initialized: now 1714551059351003 us; error 47 us; skew 500 ppm
I20240501 08:10:59.351363 30888 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:59.353567 30888 webserver.cc:460] Webserver started at http://127.28.1.131:37055/ using document root <none> and password file <none>
I20240501 08:10:59.354243 30888 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:59.354355 30888 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:59.354653 30888 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:59.357132 30888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "57210dc0a07a4851a43be23a279eb836"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.357839 30888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "57210dc0a07a4851a43be23a279eb836"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.362643 30888 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240501 08:10:59.365881 30902 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.367148 30888 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240501 08:10:59.367305 30888 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "57210dc0a07a4851a43be23a279eb836"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.367484 30888 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:59.403246 30888 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:59.404089 30888 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:59.404366 30888 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:59.405269 30888 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:59.406702 30888 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:59.406793 30888 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.406900 30888 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:59.406966 30888 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.435412 30888 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:39451
I20240501 08:10:59.435479 31014 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:39451 every 8 connection(s)
I20240501 08:10:59.436714 30888 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:10:59.445166 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 30888
I20240501 08:10:59.445336 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:10:59.448230 31015 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:10:59.448494 31015 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:59.448510 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:59.449259 31015 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:10:59.450498 30568 ts_manager.cc:194] Registered new tserver with Master: 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:10:59.451251 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:34319
I20240501 08:10:59.577335 30754 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
W20240501 08:10:59.764360 31018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:10:59.764669 31018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:10:59.764809 31018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:10:59.772234 31018 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:10:59.772389 31018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:10:59.772531 31018 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:10:59.781473 31018 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:10:59.782797 31018 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:10:59.784509 31018 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:10:59.796243 31023 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:59.796825 31026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:10:59.797355 31024 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:10:59.797950 31018 server_base.cc:1034] running on GCE node
I20240501 08:10:59.798707 31018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:10:59.799605 31018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:10:59.800797 31018 hybrid_clock.cc:648] HybridClock initialized: now 1714551059800752 us; error 43 us; skew 500 ppm
I20240501 08:10:59.801097 31018 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:10:59.803172 31018 webserver.cc:460] Webserver started at http://127.28.1.132:35637/ using document root <none> and password file <none>
I20240501 08:10:59.803874 31018 fs_manager.cc:359] Metadata directory not provided
I20240501 08:10:59.804001 31018 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:10:59.804306 31018 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:10:59.806941 31018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.807674 31018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.812537 31018 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:10:59.815627 31032 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.816908 31018 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:10:59.817054 31018 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
format_stamp: "Formatted at 2024-05-01 08:10:59 on dist-test-slave-jj4n"
I20240501 08:10:59.817234 31018 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:10:59.833520 31018 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:10:59.834348 31018 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:10:59.834611 31018 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:10:59.835500 31018 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:10:59.837008 31018 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:10:59.837108 31018 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.837211 31018 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:10:59.837284 31018 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:10:59.865655 31018 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:41135
I20240501 08:10:59.865703 31144 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:41135 every 8 connection(s)
I20240501 08:10:59.866915 31018 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:10:59.874187 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 31018
I20240501 08:10:59.874375 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:10:59.877483 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.133:0
--local_ip_for_outbound_sockets=127.28.1.133
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:10:59.878670 31145 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:10:59.878937 31145 heartbeater.cc:464] Registering TS with master...
I20240501 08:10:59.879699 31145 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:10:59.881002 30568 ts_manager.cc:194] Registered new tserver with Master: 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:10:59.881763 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:47435
I20240501 08:11:00.012858 30885 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
W20240501 08:11:00.189059 31148 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:00.189369 31148 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:11:00.189512 31148 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:00.197005 31148 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:11:00.197141 31148 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:11:00.197288 31148 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.133
I20240501 08:11:00.206465 31148 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.133
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:11:00.207751 31148 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:11:00.209527 31148 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:11:00.221582 31153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:00.221963 31154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:00.222308 31156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:11:00.223147 31148 server_base.cc:1034] running on GCE node
I20240501 08:11:00.223778 31148 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:11:00.224856 31148 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:11:00.226063 31148 hybrid_clock.cc:648] HybridClock initialized: now 1714551060225999 us; error 63 us; skew 500 ppm
I20240501 08:11:00.226419 31148 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:11:00.228816 31148 webserver.cc:460] Webserver started at http://127.28.1.133:42299/ using document root <none> and password file <none>
I20240501 08:11:00.229765 31148 fs_manager.cc:359] Metadata directory not provided
I20240501 08:11:00.229885 31148 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:11:00.230203 31148 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:11:00.232975 31148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "9204a82012ac492dbc0da9c84c51dabf"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.233753 31148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "9204a82012ac492dbc0da9c84c51dabf"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.238978 31148 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.001s sys 0.004s
I20240501 08:11:00.242372 31162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.243775 31148 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240501 08:11:00.243929 31148 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "9204a82012ac492dbc0da9c84c51dabf"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.244168 31148 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:11:00.265676 31148 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:11:00.266573 31148 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:11:00.266860 31148 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:11:00.267810 31148 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:11:00.269317 31148 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:11:00.269402 31148 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.269510 31148 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:11:00.269575 31148 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.298365 31148 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.133:42523
I20240501 08:11:00.298413 31274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.133:42523 every 8 connection(s)
I20240501 08:11:00.299671 31148 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240501 08:11:00.302513 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 31148
I20240501 08:11:00.302659 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240501 08:11:00.305488 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.134:0
--local_ip_for_outbound_sockets=127.28.1.134
--webserver_interface=127.28.1.134
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:11:00.312606 31275 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:11:00.312896 31275 heartbeater.cc:464] Registering TS with master...
I20240501 08:11:00.313664 31275 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:11:00.315048 30568 ts_manager.cc:194] Registered new tserver with Master: 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:00.315757 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.133:46157
I20240501 08:11:00.453552 31015 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
W20240501 08:11:00.623914 31278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:00.624366 31278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:11:00.624516 31278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:00.632074 31278 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:11:00.632202 31278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:11:00.632360 31278 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.134
I20240501 08:11:00.641819 31278 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.28.1.134
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.134
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:11:00.643369 31278 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:11:00.645623 31278 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:11:00.658629 31284 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:00.659019 31286 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:00.659842 31283 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:11:00.660463 31278 server_base.cc:1034] running on GCE node
I20240501 08:11:00.661136 31278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:11:00.662293 31278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:11:00.663503 31278 hybrid_clock.cc:648] HybridClock initialized: now 1714551060663449 us; error 52 us; skew 500 ppm
I20240501 08:11:00.663831 31278 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:11:00.666007 31278 webserver.cc:460] Webserver started at http://127.28.1.134:37855/ using document root <none> and password file <none>
I20240501 08:11:00.666692 31278 fs_manager.cc:359] Metadata directory not provided
I20240501 08:11:00.666785 31278 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:11:00.667058 31278 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:11:00.669550 31278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "6f37f9ed957342dba1271066c1bbeade"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.670240 31278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "6f37f9ed957342dba1271066c1bbeade"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.675184 31278 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240501 08:11:00.678403 31292 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.679713 31278 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:00.679848 31278 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "6f37f9ed957342dba1271066c1bbeade"
format_stamp: "Formatted at 2024-05-01 08:11:00 on dist-test-slave-jj4n"
I20240501 08:11:00.680078 31278 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:11:00.731029 31278 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:11:00.731842 31278 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:11:00.732120 31278 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:11:00.733050 31278 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:11:00.734460 31278 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:11:00.734544 31278 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.734625 31278 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:11:00.734676 31278 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:00.763640 31278 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.134:41579
I20240501 08:11:00.763695 31404 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.134:41579 every 8 connection(s)
I20240501 08:11:00.764930 31278 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240501 08:11:00.773270 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 31278
I20240501 08:11:00.773451 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240501 08:11:00.776471 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.135:0
--local_ip_for_outbound_sockets=127.28.1.135
--webserver_interface=127.28.1.135
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:11:00.777422 31405 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:11:00.777724 31405 heartbeater.cc:464] Registering TS with master...
I20240501 08:11:00.778594 31405 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:11:00.779884 30568 ts_manager.cc:194] Registered new tserver with Master: 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:00.780669 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.134:38199
I20240501 08:11:00.884070 31145 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
W20240501 08:11:01.093132 31408 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:01.093458 31408 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:11:01.093580 31408 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:01.100991 31408 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:11:01.101131 31408 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:11:01.101292 31408 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.135
I20240501 08:11:01.110234 31408 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.28.1.135
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.135
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:11:01.111651 31408 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:11:01.113392 31408 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:11:01.125002 31413 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:01.125648 31414 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:01.126246 31416 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:11:01.126761 31408 server_base.cc:1034] running on GCE node
I20240501 08:11:01.127327 31408 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:11:01.128265 31408 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:11:01.129446 31408 hybrid_clock.cc:648] HybridClock initialized: now 1714551061129388 us; error 50 us; skew 500 ppm
I20240501 08:11:01.129735 31408 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:11:01.131868 31408 webserver.cc:460] Webserver started at http://127.28.1.135:42973/ using document root <none> and password file <none>
I20240501 08:11:01.132594 31408 fs_manager.cc:359] Metadata directory not provided
I20240501 08:11:01.132684 31408 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:11:01.132939 31408 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:11:01.135387 31408 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
format_stamp: "Formatted at 2024-05-01 08:11:01 on dist-test-slave-jj4n"
I20240501 08:11:01.136129 31408 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
format_stamp: "Formatted at 2024-05-01 08:11:01 on dist-test-slave-jj4n"
I20240501 08:11:01.141003 31408 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240501 08:11:01.144246 31422 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:01.145650 31408 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:11:01.145803 31408 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
format_stamp: "Formatted at 2024-05-01 08:11:01 on dist-test-slave-jj4n"
I20240501 08:11:01.145992 31408 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:11:01.160933 31408 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:11:01.161779 31408 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:11:01.162034 31408 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:11:01.162938 31408 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:11:01.164394 31408 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:11:01.164487 31408 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:01.164590 31408 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:11:01.164656 31408 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:01.193230 31408 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.135:36533
I20240501 08:11:01.193285 31534 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.135:36533 every 8 connection(s)
I20240501 08:11:01.194626 31408 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240501 08:11:01.202396 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 31408
I20240501 08:11:01.202602 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240501 08:11:01.206584 31535 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:11:01.206854 31535 heartbeater.cc:464] Registering TS with master...
I20240501 08:11:01.207609 31535 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:11:01.208882 30568 ts_manager.cc:194] Registered new tserver with Master: fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533)
I20240501 08:11:01.209590 30568 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.135:59983
I20240501 08:11:01.218798 28678 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240501 08:11:01.234414 30568 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240501 08:11:01.280170 30820 tablet_service.cc:1467] Processing CreateTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 (DEFAULT_TABLE table=TestTable [id=c0be68555ded4b888585e5508b80e2db]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:01.281791 30820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:01.281708 31470 tablet_service.cc:1467] Processing CreateTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 (DEFAULT_TABLE table=TestTable [id=c0be68555ded4b888585e5508b80e2db]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:01.284575 31470 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:01.298511 30689 tablet_service.cc:1467] Processing CreateTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 (DEFAULT_TABLE table=TestTable [id=c0be68555ded4b888585e5508b80e2db]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:01.300118 30689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:01.302362 31210 tablet_service.cc:1467] Processing CreateTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 (DEFAULT_TABLE table=TestTable [id=c0be68555ded4b888585e5508b80e2db]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:01.303771 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:01.304183 30950 tablet_service.cc:1467] Processing CreateTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 (DEFAULT_TABLE table=TestTable [id=c0be68555ded4b888585e5508b80e2db]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:01.305518 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:01.305724 31557 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:01.311219 31558 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:01.312008 31557 tablet_bootstrap.cc:654] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:01.313156 31557 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:01.316004 31558 tablet_bootstrap.cc:654] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:01.317188 31558 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:01.318123 31275 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
I20240501 08:11:01.319303 31557 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:01.319617 31557 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.004s
I20240501 08:11:01.329241 31557 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.329942 31558 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: No bootstrap required, opened a new log
I20240501 08:11:01.329926 31557 raft_consensus.cc:374] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:01.330123 31557 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:01.330268 31558 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 0.019s user 0.001s sys 0.012s
I20240501 08:11:01.331095 31557 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.332713 31560 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:01.335868 31564 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:01.336691 31560 tablet_bootstrap.cc:654] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:01.336714 31562 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:01.338017 31560 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:01.340090 31562 tablet_bootstrap.cc:654] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:01.340051 31558 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.340531 31558 raft_consensus.cc:374] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:01.340633 31558 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: FOLLOWER
I20240501 08:11:01.341385 31562 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:01.341378 31558 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.344797 31557 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.025s user 0.006s sys 0.011s
I20240501 08:11:01.345820 31564 tablet_bootstrap.cc:654] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:01.347224 31564 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:01.349262 31558 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.019s user 0.012s sys 0.005s
I20240501 08:11:01.350612 31562 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:01.350888 31562 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.004s
I20240501 08:11:01.352113 31535 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
I20240501 08:11:01.354208 31560 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:01.354455 31560 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.022s user 0.001s sys 0.013s
I20240501 08:11:01.354462 31564 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:01.354719 31564 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.000s
I20240501 08:11:01.360960 31564 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.361346 31564 raft_consensus.cc:374] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:01.361449 31564 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:01.362085 31564 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.362969 31560 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.363394 31560 raft_consensus.cc:374] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:01.363478 31560 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:01.363297 31562 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.363693 31562 raft_consensus.cc:374] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:01.363744 31564 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.009s user 0.000s sys 0.010s
I20240501 08:11:01.363780 31562 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:01.364303 31560 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.364362 31562 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.366011 31560 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.011s user 0.012s sys 0.001s
I20240501 08:11:01.367122 31562 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.016s user 0.006s sys 0.005s
I20240501 08:11:01.397325 31571 raft_consensus.cc:482] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:01.397586 31571 raft_consensus.cc:504] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.399546 31571 leader_election.cc:288] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:01.411684 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:01.412462 30709 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:01.416162 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:01.416675 30840 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:01.417240 31166 leader_election.cc:302] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:01.417763 31571 raft_consensus.cc:2781] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:01.417969 31571 raft_consensus.cc:482] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:01.418082 31571 raft_consensus.cc:3037] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:01.418777 31490 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" is_pre_election: true
I20240501 08:11:01.419281 31490 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:01.420388 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:01.420771 30970 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:01.420795 31571 raft_consensus.cc:504] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.421923 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:01.422119 30840 raft_consensus.cc:3037] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:01.422106 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:01.422220 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:01.422317 30709 raft_consensus.cc:3037] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:01.422325 31490 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1469837fa8424b94b9ec4bbe57fdf740" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
I20240501 08:11:01.422377 30970 raft_consensus.cc:3037] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:01.422505 31490 raft_consensus.cc:3037] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:01.425140 30840 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:01.425320 30970 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:01.425326 30709 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:01.425727 31490 raft_consensus.cc:2454] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:01.425745 31163 leader_election.cc:302] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:01.426082 31571 leader_election.cc:288] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:01.426281 31571 raft_consensus.cc:2781] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:01.426488 31571 raft_consensus.cc:686] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Becoming Leader. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Running, Role: LEADER
I20240501 08:11:01.426980 31571 consensus_queue.cc:237] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:01.431694 30567 catalog_manager.cc:5484] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: term changed from 0 to 1, leader changed from <none> to 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133). New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } }
W20240501 08:11:01.439486 31016 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:11:01.446153 31536 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:01.448652 28678 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240501 08:11:01.450786 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 3170be3b2bbb4b3bbd8a37ac5f91de89 to finish bootstrapping
I20240501 08:11:01.459259 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 1a388864321846929919a37b570e8138 to finish bootstrapping
I20240501 08:11:01.466466 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 57210dc0a07a4851a43be23a279eb836 to finish bootstrapping
I20240501 08:11:01.473731 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 9fc0640dbc0d4208bfb536949d1bb0c4 to finish bootstrapping
I20240501 08:11:01.481854 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 9204a82012ac492dbc0da9c84c51dabf to finish bootstrapping
W20240501 08:11:01.488422 30886 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:01.489147 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6f37f9ed957342dba1271066c1bbeade to finish bootstrapping
I20240501 08:11:01.497237 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver fe99ab4507a24af4ba7fa5b5278c5ac6 to finish bootstrapping
I20240501 08:11:01.504029 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 31408
I20240501 08:11:01.504920 31530 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:11:01.551928 31276 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:11:01.557524 30755 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:01.588804 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 31408
I20240501 08:11:01.782995 31405 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
I20240501 08:11:01.844892 31571 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:01.910941 31571 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:01.918045 31165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111)
I20240501 08:11:01.921502 31588 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:01.922081 31165 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:11:04.199896 31165 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240501 08:11:05.352929 30567 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:11:05.363166 30820 tablet_service.cc:1467] Processing CreateTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.363708 30820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.364271 30689 tablet_service.cc:1467] Processing CreateTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.364737 30689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.369807 30950 tablet_service.cc:1467] Processing CreateTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.369874 30687 tablet_service.cc:1467] Processing CreateTablet for tablet 6920f9b58f814b69a3cab12f5fa3de91 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.370301 30687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6920f9b58f814b69a3cab12f5fa3de91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.370307 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.372103 30688 tablet_service.cc:1467] Processing CreateTablet for tablet bc20191679084ecfbd360e23d68abd62 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.372575 30688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.376462 30949 tablet_service.cc:1467] Processing CreateTablet for tablet 6920f9b58f814b69a3cab12f5fa3de91 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.376906 30949 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6920f9b58f814b69a3cab12f5fa3de91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.383438 30819 tablet_service.cc:1467] Processing CreateTablet for tablet bc20191679084ecfbd360e23d68abd62 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.383985 30819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.387750 31610 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:05.388343 31609 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:05.389971 31610 tablet_bootstrap.cc:654] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.391171 31609 tablet_bootstrap.cc:654] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.393860 31613 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:05.396718 31613 tablet_bootstrap.cc:654] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.397580 31610 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:05.397753 31610 ts_tablet_manager.cc:1367] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.000s
I20240501 08:11:05.398077 31210 tablet_service.cc:1467] Processing CreateTablet for tablet bc20191679084ecfbd360e23d68abd62 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.398124 31209 tablet_service.cc:1467] Processing CreateTablet for tablet 6920f9b58f814b69a3cab12f5fa3de91 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.398471 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.398708 31209 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6920f9b58f814b69a3cab12f5fa3de91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.398792 31610 raft_consensus.cc:348] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.399024 31610 raft_consensus.cc:374] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.399109 31610 raft_consensus.cc:729] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.399395 31610 consensus_queue.cc:260] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.400476 31609 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:05.400625 31609 ts_tablet_manager.cc:1367] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.003s
I20240501 08:11:05.401592 31609 raft_consensus.cc:348] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.401829 31609 raft_consensus.cc:374] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.401914 31609 raft_consensus.cc:729] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.402208 31609 consensus_queue.cc:260] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.409929 31610 ts_tablet_manager.cc:1398] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.012s user 0.005s sys 0.005s
I20240501 08:11:05.410641 31338 tablet_service.cc:1467] Processing CreateTablet for tablet 6920f9b58f814b69a3cab12f5fa3de91 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.410892 31613 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:05.411067 31613 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.003s
I20240501 08:11:05.411448 31610 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:05.411988 31338 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6920f9b58f814b69a3cab12f5fa3de91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.413087 31610 tablet_bootstrap.cc:654] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.413373 31613 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.413623 31613 raft_consensus.cc:374] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.413707 31613 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.414000 31613 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.415681 31617 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:05.417681 31617 tablet_bootstrap.cc:654] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.420272 31339 tablet_service.cc:1467] Processing CreateTablet for tablet bc20191679084ecfbd360e23d68abd62 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.420837 31339 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.409365 31340 tablet_service.cc:1467] Processing CreateTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.423306 31340 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.424492 31621 raft_consensus.cc:482] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:05.425328 31621 raft_consensus.cc:504] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.429121 31617 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:05.429360 31617 ts_tablet_manager.cc:1367] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.007s
I20240501 08:11:05.430433 31617 raft_consensus.cc:348] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.430821 31617 raft_consensus.cc:374] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.430958 31617 raft_consensus.cc:729] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.431252 31617 consensus_queue.cc:260] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.431880 31617 ts_tablet_manager.cc:1398] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:05.447376 31617 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:05.449682 31617 tablet_bootstrap.cc:654] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.462482 31621 leader_election.cc:288] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:05.474025 31617 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:05.476258 31617 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.029s user 0.002s sys 0.003s
I20240501 08:11:05.477528 31617 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.480547 31617 raft_consensus.cc:374] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.480695 31617 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.481024 31617 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.487116 31078 tablet_service.cc:1467] Processing CreateTablet for tablet 6920f9b58f814b69a3cab12f5fa3de91 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.487144 31080 tablet_service.cc:1467] Processing CreateTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.488261 31079 tablet_service.cc:1467] Processing CreateTablet for tablet bc20191679084ecfbd360e23d68abd62 (DEFAULT_TABLE table=rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:05.489491 31078 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6920f9b58f814b69a3cab12f5fa3de91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.489964 31080 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.491286 31079 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:05.496054 31609 ts_tablet_manager.cc:1398] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.095s user 0.014s sys 0.025s
I20240501 08:11:05.496573 31609 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:05.498384 31609 tablet_bootstrap.cc:654] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.503057 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:05.503299 30970 raft_consensus.cc:2454] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:05.503583 31609 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:05.503870 31609 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:11:05.505888 31610 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:05.506147 31610 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.095s user 0.014s sys 0.023s
I20240501 08:11:05.507006 31610 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.507392 31610 raft_consensus.cc:374] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.507508 31610 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.507825 31610 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.508587 31610 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:05.504815 31609 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.509150 31609 raft_consensus.cc:374] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.509274 31609 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.509588 31609 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.510202 31609 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:11:05.510579 31609 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:05.512459 31609 tablet_bootstrap.cc:654] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.521256 31617 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.044s user 0.001s sys 0.001s
I20240501 08:11:05.521433 31609 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:05.521582 31609 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.004s
I20240501 08:11:05.522344 31609 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.522635 31609 raft_consensus.cc:374] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.522750 31609 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.523057 31609 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.523699 31609 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:11:05.524020 31630 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:05.523583 31628 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:05.530066 31613 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.119s user 0.048s sys 0.048s
I20240501 08:11:05.530450 31613 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:05.532068 31613 tablet_bootstrap.cc:654] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
W20240501 08:11:05.532471 30643 leader_election.cc:341] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Not found: Tablet not found: 6920f9b58f814b69a3cab12f5fa3de91
I20240501 08:11:05.535501 31628 tablet_bootstrap.cc:654] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.536166 31630 tablet_bootstrap.cc:654] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.536789 31613 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:05.537006 31613 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240501 08:11:05.537921 31613 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.538234 31613 raft_consensus.cc:374] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.538340 31613 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.538681 31613 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.539019 31630 log.cc:824] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:05.535995 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:05.539388 31613 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:05.539638 31628 log.cc:824] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: Log is configured to *not* fsync() on all Append() calls
W20240501 08:11:05.540055 30644 leader_election.cc:341] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:05.541200 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:05.541608 31230 raft_consensus.cc:2454] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:05.542273 30643 leader_election.cc:302] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 5 voters: 3 yes votes; 2 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters: 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:11:05.542819 31616 raft_consensus.cc:2781] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:05.542924 31616 raft_consensus.cc:482] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:05.543042 31616 raft_consensus.cc:3037] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.543936 31628 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:05.544229 31628 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.021s user 0.009s sys 0.000s
I20240501 08:11:05.545328 31630 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:05.545606 31630 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.023s user 0.004s sys 0.003s
I20240501 08:11:05.545836 31616 raft_consensus.cc:504] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.546917 31616 leader_election.cc:288] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Requested vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:05.547227 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:05.547459 30970 raft_consensus.cc:3037] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.547261 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:05.549613 31628 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.550019 31628 raft_consensus.cc:374] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.550132 31628 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.550166 30970 raft_consensus.cc:2454] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
I20240501 08:11:05.550457 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
W20240501 08:11:05.550947 30643 leader_election.cc:341] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:05.550881 31628 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.551421 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:05.551610 31230 raft_consensus.cc:3037] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.552467 31628 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.008s user 0.008s sys 0.000s
I20240501 08:11:05.552845 31628 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
W20240501 08:11:05.553182 30644 leader_election.cc:341] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:05.554438 31628 tablet_bootstrap.cc:654] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.554908 31230 raft_consensus.cc:2454] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
I20240501 08:11:05.555723 30643 leader_election.cc:302] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 5 voters: 3 yes votes; 2 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters: 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:11:05.557153 31616 raft_consensus.cc:2781] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:05.557891 31616 raft_consensus.cc:686] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Becoming Leader. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Running, Role: LEADER
I20240501 08:11:05.558449 31616 consensus_queue.cc:237] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.561960 31630 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.562546 31630 raft_consensus.cc:374] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.562692 31630 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.563589 31630 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.566716 31628 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:05.566915 31628 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.010s
I20240501 08:11:05.567683 31630 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.022s user 0.009s sys 0.001s
I20240501 08:11:05.567878 31628 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.568152 31630 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:05.567668 30566 catalog_manager.cc:5484] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:05.568363 31628 raft_consensus.cc:374] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.568495 31628 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.568809 31628 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.569406 31628 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:05.569706 31630 tablet_bootstrap.cc:654] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.571573 31628 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:05.574014 31628 tablet_bootstrap.cc:654] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.574465 31630 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:05.574604 31630 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.005s
I20240501 08:11:05.575445 31630 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.575690 31630 raft_consensus.cc:374] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.575779 31630 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.576056 31630 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.576231 31628 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:05.576426 31628 ts_tablet_manager.cc:1367] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
I20240501 08:11:05.576531 31630 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:05.576793 31630 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:05.577209 31628 raft_consensus.cc:348] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.577463 31628 raft_consensus.cc:374] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.577545 31628 raft_consensus.cc:729] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.577832 31628 consensus_queue.cc:260] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.578069 31630 tablet_bootstrap.cc:654] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:05.578893 31628 ts_tablet_manager.cc:1398] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:05.580978 31630 tablet_bootstrap.cc:492] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:05.581120 31630 ts_tablet_manager.cc:1367] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240501 08:11:05.581781 31630 raft_consensus.cc:348] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.582044 31630 raft_consensus.cc:374] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:05.582134 31630 raft_consensus.cc:729] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:05.582391 31630 consensus_queue.cc:260] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.582835 31630 ts_tablet_manager.cc:1398] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:11:05.587500 31596 raft_consensus.cc:482] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:05.587677 31596 raft_consensus.cc:504] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.588856 31596 leader_election.cc:288] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:05.589020 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:05.589210 30840 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:05.589387 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:05.589619 30709 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:05.590059 31163 leader_election.cc:302] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:05.590423 31596 raft_consensus.cc:2781] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:05.590561 31596 raft_consensus.cc:482] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:05.590672 31596 raft_consensus.cc:3037] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.593588 31596 raft_consensus.cc:504] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.594846 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:05.595072 30840 raft_consensus.cc:3037] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.595674 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:05.596056 31100 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:05.596179 31596 leader_election.cc:288] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:05.596380 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:05.596693 31360 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:05.596768 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:05.596881 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:05.596944 31100 raft_consensus.cc:3037] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.597107 30709 raft_consensus.cc:3037] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.597281 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc20191679084ecfbd360e23d68abd62" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:05.597434 31360 raft_consensus.cc:3037] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.597608 30840 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:05.599141 31100 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:05.599486 31164 leader_election.cc:302] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:05.599761 31596 raft_consensus.cc:2781] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:05.599767 30709 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:05.600170 31360 raft_consensus.cc:2454] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:05.600499 31596 raft_consensus.cc:686] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Becoming Leader. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Running, Role: LEADER
I20240501 08:11:05.600818 31596 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.603739 30568 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: term changed from 0 to 1, leader changed from <none> to 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133). New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
W20240501 08:11:05.622267 31146 tablet.cc:2369] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:05.653653 31639 raft_consensus.cc:482] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:05.653877 31639 raft_consensus.cc:504] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.655751 31639 leader_election.cc:288] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:05.666106 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:05.666388 31360 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:05.668706 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:05.668969 30840 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:05.669430 31036 leader_election.cc:302] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:05.669878 31639 raft_consensus.cc:2781] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:05.669991 31639 raft_consensus.cc:482] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:05.670068 31639 raft_consensus.cc:3037] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.670382 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:05.670722 30709 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:05.671499 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:05.671713 30970 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:05.672817 31639 raft_consensus.cc:504] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.673576 31639 leader_election.cc:288] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:05.673817 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:05.673909 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:05.674010 30840 raft_consensus.cc:3037] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.674115 31360 raft_consensus.cc:3037] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.674150 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:05.674221 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1280b2f88bbc470bbc9bc49c95a4b108" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:05.674314 30709 raft_consensus.cc:3037] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.674379 30970 raft_consensus.cc:3037] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.676698 30840 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 1.
I20240501 08:11:05.676792 31360 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 1.
I20240501 08:11:05.676824 30709 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 1.
I20240501 08:11:05.676896 30970 raft_consensus.cc:2454] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 1.
I20240501 08:11:05.677211 31035 leader_election.cc:302] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:05.677476 31639 raft_consensus.cc:2781] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:05.677912 31639 raft_consensus.cc:686] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 LEADER]: Becoming Leader. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Running, Role: LEADER
I20240501 08:11:05.678537 31639 consensus_queue.cc:237] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:05.683180 30568 catalog_manager.cc:5484] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132). New cstate: current_term: 1 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } }
W20240501 08:11:05.769539 31406 tablet.cc:2369] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:05.930334 31639 raft_consensus.cc:482] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:05.930588 31639 raft_consensus.cc:504] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:05.931857 31639 leader_election.cc:288] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:05.932286 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:05.932286 30709 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:05.932595 30970 raft_consensus.cc:2379] T 6920f9b58f814b69a3cab12f5fa3de91 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in current term 1: Already voted for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in this term.
I20240501 08:11:05.932586 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:05.932832 31360 raft_consensus.cc:2454] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:05.939779 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6920f9b58f814b69a3cab12f5fa3de91" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:05.940124 31230 raft_consensus.cc:2379] T 6920f9b58f814b69a3cab12f5fa3de91 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in current term 1: Already voted for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in this term.
I20240501 08:11:05.940677 31034 leader_election.cc:302] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf
I20240501 08:11:05.940984 31639 raft_consensus.cc:3037] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:05.942917 31639 raft_consensus.cc:2726] T 6920f9b58f814b69a3cab12f5fa3de91 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240501 08:11:06.011684 31654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:06.012038 31654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:06.019793 31654 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:11:06.037025 31596 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.054477 31596 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
I20240501 08:11:06.058874 31616 consensus_queue.cc:1035] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.072571 31644 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.073021 31644 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.076215 31360 raft_consensus.cc:3037] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.080386 31616 consensus_queue.cc:1035] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.100124 31639 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.114490 31616 consensus_queue.cc:1035] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.114990 31616 consensus_queue.cc:1035] T 6920f9b58f814b69a3cab12f5fa3de91 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.120106 31652 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:06.147778 31652 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240501 08:11:06.150161 31652 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:06.448099 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 55665.8us
INSERT report
rows total: 2000
time total: 383.326 ms
time per row: 0.191663 ms
I20240501 08:11:06.632779 30563 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:11:06.641127 30949 tablet_service.cc:1467] Processing CreateTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.641350 30819 tablet_service.cc:1467] Processing CreateTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.641607 30949 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.641719 30819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.642741 31339 tablet_service.cc:1467] Processing CreateTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.643164 31339 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.643132 31209 tablet_service.cc:1467] Processing CreateTablet for tablet 5730faa05092410da14e00e24ad74c86 (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.643522 31209 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.644698 30950 tablet_service.cc:1467] Processing CreateTablet for tablet 5730faa05092410da14e00e24ad74c86 (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.645037 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.648403 31078 tablet_service.cc:1467] Processing CreateTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.648821 31078 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.650224 30948 tablet_service.cc:1467] Processing CreateTablet for tablet e61aa5646d3947c7b68bef19ee280c6e (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.650719 30948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.652356 31079 tablet_service.cc:1467] Processing CreateTablet for tablet 5730faa05092410da14e00e24ad74c86 (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.652701 31079 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.652745 30687 tablet_service.cc:1467] Processing CreateTablet for tablet e61aa5646d3947c7b68bef19ee280c6e (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.653129 30687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.655323 31210 tablet_service.cc:1467] Processing CreateTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.655683 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.659935 31208 tablet_service.cc:1467] Processing CreateTablet for tablet e61aa5646d3947c7b68bef19ee280c6e (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.660424 31208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.665051 30820 tablet_service.cc:1467] Processing CreateTablet for tablet 5730faa05092410da14e00e24ad74c86 (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.665149 30818 tablet_service.cc:1467] Processing CreateTablet for tablet e61aa5646d3947c7b68bef19ee280c6e (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.665537 30820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.665772 30818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.666489 31338 tablet_service.cc:1467] Processing CreateTablet for tablet 5730faa05092410da14e00e24ad74c86 (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.666915 31338 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.667748 31750 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:06.669765 31750 tablet_bootstrap.cc:654] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.671301 31748 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:06.671535 31080 tablet_service.cc:1467] Processing CreateTablet for tablet e61aa5646d3947c7b68bef19ee280c6e (DEFAULT_TABLE table=rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:06.672082 31080 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:06.673328 31748 tablet_bootstrap.cc:654] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.675561 31750 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:06.675685 31747 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:06.675712 31750 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240501 08:11:06.676476 31750 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.676740 31750 raft_consensus.cc:374] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.676828 31750 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.677070 31750 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.677511 31747 tablet_bootstrap.cc:654] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.678440 31750 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:06.678704 31750 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:06.680364 31748 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:06.681241 31750 tablet_bootstrap.cc:654] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.681872 31748 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.001s
I20240501 08:11:06.682844 31747 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:06.682991 31747 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.003s
I20240501 08:11:06.683683 31747 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.682746 31748 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.684545 31748 raft_consensus.cc:374] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.684684 31748 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.684944 31748 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.685709 31748 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240501 08:11:06.683939 31747 raft_consensus.cc:374] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.686523 31747 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.686970 31747 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.692245 31748 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:06.694113 31748 tablet_bootstrap.cc:654] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.694290 31747 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.011s user 0.005s sys 0.000s
I20240501 08:11:06.695410 31747 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:06.699265 31747 tablet_bootstrap.cc:654] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.704104 31748 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:06.704355 31748 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.006s
I20240501 08:11:06.705192 31748 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.705575 31748 raft_consensus.cc:374] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.705687 31748 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.705894 31747 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:06.706066 31747 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.000s
I20240501 08:11:06.706035 31748 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.706701 31748 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:06.707118 31748 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:06.710216 31747 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.710502 31747 raft_consensus.cc:374] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.710566 31747 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.711975 31748 tablet_bootstrap.cc:654] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.711978 31747 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.712579 31756 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:06.713903 31747 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.008s user 0.001s sys 0.002s
I20240501 08:11:06.714174 31747 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:06.714339 31756 tablet_bootstrap.cc:654] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.716897 31755 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:06.718456 31747 tablet_bootstrap.cc:654] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.718534 31755 tablet_bootstrap.cc:654] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.719942 31749 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:06.720412 31748 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:06.720685 31748 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.014s user 0.007s sys 0.000s
I20240501 08:11:06.721535 31747 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:06.721686 31747 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.001s
I20240501 08:11:06.722193 31756 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:06.722358 31756 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20240501 08:11:06.722128 31748 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.722450 31747 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.722697 31747 raft_consensus.cc:374] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.722649 31748 raft_consensus.cc:374] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.722791 31747 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.722836 31748 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.723055 31747 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.723361 31749 tablet_bootstrap.cc:654] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.723188 31748 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.723842 31756 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.724110 31756 raft_consensus.cc:374] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.724198 31756 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.725036 31748 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:11:06.726683 31747 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.005s user 0.002s sys 0.001s
I20240501 08:11:06.727591 31755 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:06.727800 31755 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.000s
I20240501 08:11:06.727716 31756 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.728598 31755 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.728760 31764 raft_consensus.cc:482] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:06.728919 31755 raft_consensus.cc:374] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.729076 31755 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.728938 31764 raft_consensus.cc:504] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.729522 31750 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:06.729434 31755 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.729696 31750 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.051s user 0.016s sys 0.026s
I20240501 08:11:06.730414 31750 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.730661 31750 raft_consensus.cc:374] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.730892 31755 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:06.731204 31755 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:06.731462 31749 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:06.731631 31749 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.004s
I20240501 08:11:06.732389 31749 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.732636 31749 raft_consensus.cc:374] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.732739 31749 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.732750 31750 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.732964 31749 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.733024 31750 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.733562 31749 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:06.734063 31756 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.012s user 0.004s sys 0.000s
I20240501 08:11:06.734393 31756 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:06.734630 31755 tablet_bootstrap.cc:654] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.735066 31750 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240501 08:11:06.735345 31750 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:06.736281 31756 tablet_bootstrap.cc:654] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.737898 31750 tablet_bootstrap.cc:654] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.744321 31755 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:06.744588 31755 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.007s
I20240501 08:11:06.746030 31755 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.746430 31755 raft_consensus.cc:374] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.746543 31755 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.746886 31755 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.748371 31755 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240501 08:11:06.748718 31755 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:06.751163 31755 tablet_bootstrap.cc:654] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:06.753543 31750 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:06.753715 31750 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.018s user 0.006s sys 0.009s
I20240501 08:11:06.754770 31750 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.755051 31750 raft_consensus.cc:374] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.755133 31750 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.755424 31750 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.756816 31750 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:11:06.757144 31755 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:06.757275 31755 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.003s
I20240501 08:11:06.757887 31755 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.758129 31755 raft_consensus.cc:374] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.758213 31755 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.758432 31755 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.759481 31755 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:06.762620 31696 raft_consensus.cc:482] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:06.762768 31696 raft_consensus.cc:504] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.763684 31696 leader_election.cc:288] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637)
I20240501 08:11:06.763916 31596 raft_consensus.cc:482] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:06.764110 31596 raft_consensus.cc:504] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.764468 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:06.764725 30970 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:06.764983 31596 leader_election.cc:288] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:06.765153 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:06.765393 30708 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:06.765959 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:06.766122 30708 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.766315 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:06.766522 30970 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.767016 31163 leader_election.cc:302] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:06.767068 31033 leader_election.cc:302] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:06.767297 31596 raft_consensus.cc:2781] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:06.767367 31596 raft_consensus.cc:482] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:06.767381 31696 raft_consensus.cc:2781] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:06.767443 31596 raft_consensus.cc:3037] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.767479 31696 raft_consensus.cc:482] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:06.767554 31696 raft_consensus.cc:3037] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.770118 31596 raft_consensus.cc:504] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.771234 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:06.770514 31696 raft_consensus.cc:504] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.771260 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:06.771477 31230 raft_consensus.cc:2379] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.771760 31099 raft_consensus.cc:2379] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9204a82012ac492dbc0da9c84c51dabf in current term 1: Already voted for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in this term.
I20240501 08:11:06.772514 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:06.772703 30708 raft_consensus.cc:3037] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.772770 30970 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:06.773006 30970 raft_consensus.cc:3037] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.775488 30708 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.776149 31644 raft_consensus.cc:482] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:06.776180 31596 leader_election.cc:288] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:06.776362 31644 raft_consensus.cc:504] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.777340 31644 leader_election.cc:288] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:06.779138 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:06.779350 31100 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.780144 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:06.780419 31359 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.782099 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:06.782382 31099 raft_consensus.cc:2379] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 9204a82012ac492dbc0da9c84c51dabf in current term 1: Already voted for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in this term.
I20240501 08:11:06.782748 31164 leader_election.cc:302] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:06.783198 31644 raft_consensus.cc:2781] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:06.783310 31644 raft_consensus.cc:482] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:06.783377 31644 raft_consensus.cc:3037] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.785457 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:06.785741 31230 raft_consensus.cc:2379] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.786087 31644 raft_consensus.cc:504] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.787662 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:06.787729 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:06.787873 31099 raft_consensus.cc:3037] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.788137 30708 raft_consensus.cc:2379] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.788522 31644 leader_election.cc:288] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:06.790814 31099 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.791679 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:06.791872 31359 raft_consensus.cc:3037] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.791872 31696 leader_election.cc:288] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 election: Requested vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637)
I20240501 08:11:06.777797 30968 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:06.793231 30968 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.786962 30969 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:06.787744 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:06.793994 30967 raft_consensus.cc:3037] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.794771 31359 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.796277 31033 leader_election.cc:302] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 9fc0640dbc0d4208bfb536949d1bb0c4; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf
I20240501 08:11:06.775914 30970 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.796957 30967 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.796911 31163 leader_election.cc:302] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters: 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:11:06.797302 31165 leader_election.cc:302] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:06.797623 31596 raft_consensus.cc:2781] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:06.797355 31644 raft_consensus.cc:2781] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:06.798121 31696 raft_consensus.cc:2726] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240501 08:11:06.803884 31644 raft_consensus.cc:686] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Becoming Leader. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Running, Role: LEADER
I20240501 08:11:06.804308 31644 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:06.809070 31596 raft_consensus.cc:686] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Becoming Leader. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Running, Role: LEADER
I20240501 08:11:06.809434 31596 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:06.809744 31764 leader_election.cc:288] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:06.817920 30839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:06.818045 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:06.818169 30838 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:06.818318 30839 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.818454 30838 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:06.818889 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:06.819201 30836 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
I20240501 08:11:06.819428 30835 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:06.819763 30835 raft_consensus.cc:3037] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.819536 30837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:06.820735 30837 raft_consensus.cc:3037] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.832697 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:06.833029 30967 raft_consensus.cc:2379] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6f37f9ed957342dba1271066c1bbeade in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.834678 30837 raft_consensus.cc:2454] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:06.834986 30835 raft_consensus.cc:2454] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 1.
I20240501 08:11:06.836216 31756 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:06.836428 31756 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.102s user 0.036s sys 0.050s
I20240501 08:11:06.837168 30835 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:06.837477 30835 raft_consensus.cc:2379] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6f37f9ed957342dba1271066c1bbeade in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.838003 31756 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.838275 31756 raft_consensus.cc:374] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:06.838371 31756 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:06.838629 31756 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.839222 31756 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240501 08:11:06.841811 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:06.842093 31099 raft_consensus.cc:2379] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 6f37f9ed957342dba1271066c1bbeade in current term 1: Already voted for candidate 9204a82012ac492dbc0da9c84c51dabf in this term.
I20240501 08:11:06.842555 31294 leader_election.cc:302] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 6f37f9ed957342dba1271066c1bbeade; no voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:11:06.842325 30562 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: term changed from 0 to 1, leader changed from <none> to 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133). New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:06.843815 31764 raft_consensus.cc:2726] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:11:06.845710 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5730faa05092410da14e00e24ad74c86" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:06.847108 30562 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: term changed from 0 to 1, leader changed from <none> to 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133). New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:06.860863 31764 raft_consensus.cc:482] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:06.861048 31764 raft_consensus.cc:504] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.861856 31764 leader_election.cc:288] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:06.862180 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:06.862212 30835 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:06.862305 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:06.862428 30967 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 0.
I20240501 08:11:06.862462 30835 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 0.
I20240501 08:11:06.862584 31099 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 0.
I20240501 08:11:06.862823 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:06.862967 31294 leader_election.cc:302] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:06.863003 31230 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 0.
I20240501 08:11:06.863226 31764 raft_consensus.cc:2781] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:06.863355 31764 raft_consensus.cc:482] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:06.863461 31764 raft_consensus.cc:3037] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.865484 31764 raft_consensus.cc:504] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.866362 31764 leader_election.cc:288] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 election: Requested vote from peers 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:06.866530 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:06.866601 30835 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:06.866721 30967 raft_consensus.cc:3037] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.866806 30835 raft_consensus.cc:3037] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.868600 30967 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 1.
I20240501 08:11:06.869282 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:06.869504 31099 raft_consensus.cc:3037] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.869537 30835 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 1.
I20240501 08:11:06.869694 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" candidate_uuid: "6f37f9ed957342dba1271066c1bbeade" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:06.869894 31230 raft_consensus.cc:3037] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:06.870054 31296 leader_election.cc:302] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade; no voters:
I20240501 08:11:06.870395 31764 raft_consensus.cc:2781] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:06.870682 31764 raft_consensus.cc:686] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 LEADER]: Becoming Leader. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Running, Role: LEADER
I20240501 08:11:06.871241 31764 consensus_queue.cc:237] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:06.872306 31099 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 1.
I20240501 08:11:06.872611 31230 raft_consensus.cc:2454] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f37f9ed957342dba1271066c1bbeade in term 1.
I20240501 08:11:06.876164 30562 catalog_manager.cc:5484] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade reported cstate change: term changed from 0 to 1, leader changed from <none> to 6f37f9ed957342dba1271066c1bbeade (127.28.1.134). New cstate: current_term: 1 leader_uuid: "6f37f9ed957342dba1271066c1bbeade" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } }
W20240501 08:11:06.959424 31165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240501 08:11:06.960722 31165 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240501 08:11:07.182330 31596 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.196240 31596 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.202865 31789 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.203524 31789 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:07.247467 31783 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:07.247812 31783 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:07.256306 31783 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:11:07.262145 31789 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.262415 31596 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.269129 31773 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.269408 31774 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_1'
I20240501 08:11:07.359905 31764 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.424533 31764 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.467881 31764 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:07.539216 31764 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:07.575786 31296 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 59710us
W20240501 08:11:07.576088 31293 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 86063.4us
W20240501 08:11:07.588894 31294 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 76335.5us
I20240501 08:11:07.633183 31853 mvcc.cc:205] Tried to move back new op lower bound from 7022801172431400960 to 7022801169912455168. Current Snapshot: MvccSnapshot[applied={T|T < 7022801172431400960}]
INSERT report
rows total: 2000
time total: 317.613 ms
time per row: 0.158806 ms
I20240501 08:11:07.734458 30566 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:11:07.743016 30687 tablet_service.cc:1467] Processing CreateTablet for tablet e2393202bb014986b2496d3c99c65962 (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.743501 30687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2393202bb014986b2496d3c99c65962. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.743461 30950 tablet_service.cc:1467] Processing CreateTablet for tablet e2393202bb014986b2496d3c99c65962 (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.743847 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2393202bb014986b2496d3c99c65962. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.744632 30948 tablet_service.cc:1467] Processing CreateTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.745025 30948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.745375 30820 tablet_service.cc:1467] Processing CreateTablet for tablet e2393202bb014986b2496d3c99c65962 (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.745774 30820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2393202bb014986b2496d3c99c65962. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.748437 30689 tablet_service.cc:1467] Processing CreateTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.748824 30689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.749198 31208 tablet_service.cc:1467] Processing CreateTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.749339 31338 tablet_service.cc:1467] Processing CreateTablet for tablet e2393202bb014986b2496d3c99c65962 (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.749625 31208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.749745 31338 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2393202bb014986b2496d3c99c65962. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.752329 31339 tablet_service.cc:1467] Processing CreateTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.752705 31339 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.752641 30688 tablet_service.cc:1467] Processing CreateTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.752992 30688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.752969 31079 tablet_service.cc:1467] Processing CreateTablet for tablet e2393202bb014986b2496d3c99c65962 (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.753437 31079 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e2393202bb014986b2496d3c99c65962. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.756314 31340 tablet_service.cc:1467] Processing CreateTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.756657 31340 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.757083 30819 tablet_service.cc:1467] Processing CreateTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.757505 30819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.775943 31867 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:07.776528 31868 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:07.777058 31080 tablet_service.cc:1467] Processing CreateTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.777518 31080 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.778324 30818 tablet_service.cc:1467] Processing CreateTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.778728 30818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.779929 31868 tablet_bootstrap.cc:654] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.786170 31210 tablet_service.cc:1467] Processing CreateTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae (DEFAULT_TABLE table=rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:07.786670 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:07.789330 31868 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:07.789969 31868 ts_tablet_manager.cc:1367] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.014s user 0.004s sys 0.000s
I20240501 08:11:07.791049 31868 raft_consensus.cc:348] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.796470 31868 raft_consensus.cc:374] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.796789 31868 raft_consensus.cc:729] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.797277 31868 consensus_queue.cc:260] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.797983 31867 tablet_bootstrap.cc:654] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.799194 31868 ts_tablet_manager.cc:1398] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240501 08:11:07.799923 31868 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:07.802350 31865 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:07.803653 31868 tablet_bootstrap.cc:654] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.804808 31870 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:07.806054 31865 tablet_bootstrap.cc:654] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.806736 31870 tablet_bootstrap.cc:654] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.809033 31871 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:07.811385 31866 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:07.811481 31696 raft_consensus.cc:482] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:07.811726 31696 raft_consensus.cc:504] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.812070 31871 tablet_bootstrap.cc:654] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.812781 31696 leader_election.cc:288] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:11:07.813263 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:07.813838 31868 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:07.813994 31868 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.009s
I20240501 08:11:07.813983 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:07.814182 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:07.814651 31866 tablet_bootstrap.cc:654] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
W20240501 08:11:07.814713 31035 leader_election.cc:341] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:07.814749 31868 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.815029 31868 raft_consensus.cc:374] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.815101 31868 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.815335 31868 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.815418 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
W20240501 08:11:07.815524 31036 leader_election.cc:341] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:11:07.815891 31033 leader_election.cc:341] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:07.816023 31033 leader_election.cc:302] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 9fc0640dbc0d4208bfb536949d1bb0c4; no voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 6f37f9ed957342dba1271066c1bbeade
W20240501 08:11:07.816442 31033 leader_election.cc:341] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:07.816520 31868 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:07.817414 31696 raft_consensus.cc:2726] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:11:07.817590 31871 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:07.817781 31871 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240501 08:11:07.818185 31867 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:07.818483 31867 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.043s user 0.007s sys 0.000s
I20240501 08:11:07.818706 31871 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.819008 31871 raft_consensus.cc:374] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.819151 31871 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.819202 31870 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:07.819422 31867 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.819660 31867 raft_consensus.cc:374] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.819468 31871 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.819747 31867 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.820169 31870 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.000s
I20240501 08:11:07.820004 31867 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.820407 31871 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:07.820890 31870 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.821105 31870 raft_consensus.cc:374] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.821182 31870 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.821383 31867 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:07.821379 31870 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.821647 31867 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:07.822639 31870 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:07.822857 31871 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:07.822898 31870 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:07.823575 31865 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:07.823731 31865 ts_tablet_manager.cc:1367] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.022s user 0.000s sys 0.004s
I20240501 08:11:07.824549 31865 raft_consensus.cc:348] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.824784 31865 raft_consensus.cc:374] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.824867 31865 raft_consensus.cc:729] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.825897 31871 tablet_bootstrap.cc:654] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.825922 31867 tablet_bootstrap.cc:654] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.826855 31865 consensus_queue.cc:260] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.827625 31865 ts_tablet_manager.cc:1398] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240501 08:11:07.827894 31865 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:07.829522 31865 tablet_bootstrap.cc:654] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.829653 31870 tablet_bootstrap.cc:654] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.832835 31867 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:07.832861 31866 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:07.833003 31866 ts_tablet_manager.cc:1367] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.022s user 0.003s sys 0.000s
I20240501 08:11:07.833029 31867 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.000s
I20240501 08:11:07.833738 31866 raft_consensus.cc:348] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.833977 31866 raft_consensus.cc:374] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.833865 31867 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.834057 31866 raft_consensus.cc:729] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.834126 31867 raft_consensus.cc:374] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.834206 31867 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.834295 31866 consensus_queue.cc:260] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.834492 31867 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.835486 31866 ts_tablet_manager.cc:1398] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:07.835779 31866 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:07.835892 31867 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:11:07.836210 31867 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:07.837431 31866 tablet_bootstrap.cc:654] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.839090 31867 tablet_bootstrap.cc:654] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.841130 31865 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:07.841145 31870 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:07.841320 31865 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240501 08:11:07.841334 31870 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.000s
I20240501 08:11:07.842157 31870 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.842222 31865 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.842429 31870 raft_consensus.cc:374] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.842487 31870 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.842512 31865 raft_consensus.cc:374] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.842582 31865 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.842715 31870 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.842839 31865 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.843250 31870 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:07.843593 31870 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:07.844354 31865 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:07.844637 31865 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:07.844834 31867 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:07.845012 31867 ts_tablet_manager.cc:1367] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.003s
I20240501 08:11:07.845788 31867 raft_consensus.cc:348] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.846017 31870 tablet_bootstrap.cc:654] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.846118 31865 tablet_bootstrap.cc:654] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:07.846132 31867 raft_consensus.cc:374] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.846303 31867 raft_consensus.cc:729] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.846626 31867 consensus_queue.cc:260] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.847242 31867 ts_tablet_manager.cc:1398] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:07.848493 31871 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:07.848722 31871 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.026s user 0.003s sys 0.016s
I20240501 08:11:07.848750 31866 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:07.848901 31866 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.000s
I20240501 08:11:07.849648 31866 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.849650 31871 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.849910 31866 raft_consensus.cc:374] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.849995 31866 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.850024 31871 raft_consensus.cc:374] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.850178 31871 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.850275 31866 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.850529 31871 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.850844 31866 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:07.851295 31871 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:07.852818 31865 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:07.853041 31865 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.002s
I20240501 08:11:07.853804 31865 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.854090 31865 raft_consensus.cc:374] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.854204 31865 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.854465 31865 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.855026 31865 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:07.858170 31870 tablet_bootstrap.cc:492] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:07.858353 31870 ts_tablet_manager.cc:1367] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.005s
I20240501 08:11:07.859124 31870 raft_consensus.cc:348] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.859347 31870 raft_consensus.cc:374] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:07.859443 31870 raft_consensus.cc:729] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:07.859648 31870 consensus_queue.cc:260] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.860143 31870 ts_tablet_manager.cc:1398] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:07.873514 31773 raft_consensus.cc:482] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:07.873736 31773 raft_consensus.cc:504] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.874658 31773 leader_election.cc:288] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:07.875038 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:07.875067 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:07.875280 30836 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:07.875290 30708 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:07.875308 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:07.875339 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:07.875502 31359 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:07.875560 30967 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 0.
I20240501 08:11:07.875769 31166 leader_election.cc:302] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:07.876055 31773 raft_consensus.cc:2781] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:07.876154 31773 raft_consensus.cc:482] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:07.876255 31773 raft_consensus.cc:3037] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.878516 31773 raft_consensus.cc:504] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.879601 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:07.879616 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:07.879734 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:07.879812 30967 raft_consensus.cc:3037] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.879835 30836 raft_consensus.cc:3037] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.879912 31359 raft_consensus.cc:3037] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.882611 30967 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:07.882612 30836 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:07.882622 31359 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:07.883138 31773 leader_election.cc:288] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Requested vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:07.883169 31166 leader_election.cc:302] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:07.883409 31773 raft_consensus.cc:2781] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:07.883352 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b8a23ca81da4039b7c18c84b2f2d8fb" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:07.883538 30708 raft_consensus.cc:3037] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.883816 31773 raft_consensus.cc:686] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Becoming Leader. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Running, Role: LEADER
I20240501 08:11:07.884136 31773 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:07.886405 30708 raft_consensus.cc:2454] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9204a82012ac492dbc0da9c84c51dabf in term 1.
I20240501 08:11:07.887807 30566 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: term changed from 0 to 1, leader changed from <none> to 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133). New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:07.888613 31757 raft_consensus.cc:482] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:07.888779 31757 raft_consensus.cc:504] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.891002 31757 leader_election.cc:288] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:07.906239 31757 raft_consensus.cc:482] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:07.906455 31757 raft_consensus.cc:504] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.907646 31757 leader_election.cc:288] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:07.908062 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:07.908084 31099 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:07.908335 31100 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.908335 31099 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.912732 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:07.913003 31359 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.913323 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:07.913549 31360 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.914232 30774 leader_election.cc:302] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:07.914716 31757 raft_consensus.cc:2781] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:07.914813 31757 raft_consensus.cc:482] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:07.914907 31757 raft_consensus.cc:3037] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.915668 30774 leader_election.cc:302] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:07.918088 31757 raft_consensus.cc:504] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.919351 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:07.919567 31360 raft_consensus.cc:3037] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.919890 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:07.919862 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:07.920120 30967 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.920167 30708 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.920805 31896 raft_consensus.cc:2781] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:07.920959 31896 raft_consensus.cc:482] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:07.921105 31896 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.921160 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:07.921411 31757 leader_election.cc:288] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Requested vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:07.921401 31230 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.922070 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:07.922170 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:07.922226 31360 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.922278 30708 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:07.922338 30967 raft_consensus.cc:3037] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.922853 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:07.923089 30708 raft_consensus.cc:3037] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.923482 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e2393202bb014986b2496d3c99c65962" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:07.923676 31100 raft_consensus.cc:3037] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.924904 30967 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.925328 30773 leader_election.cc:302] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade; no voters:
I20240501 08:11:07.925729 31757 raft_consensus.cc:2781] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:07.926067 30708 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.926322 31100 raft_consensus.cc:2454] T e2393202bb014986b2496d3c99c65962 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.925725 31896 raft_consensus.cc:504] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.926942 31757 raft_consensus.cc:686] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Becoming Leader. State: Replica: 1a388864321846929919a37b570e8138, State: Running, Role: LEADER
I20240501 08:11:07.927309 31896 leader_election.cc:288] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Requested vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:07.927488 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:07.927488 31757 consensus_queue.cc:237] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:07.927683 31360 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.928022 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:07.928248 31230 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.927806 30708 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:07.928568 30708 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.930374 31360 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.930914 31230 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.930899 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:07.931150 31100 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:07.931353 30708 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.931730 30775 leader_election.cc:302] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:07.933347 31897 raft_consensus.cc:2781] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:07.933502 31897 raft_consensus.cc:686] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Becoming Leader. State: Replica: 1a388864321846929919a37b570e8138, State: Running, Role: LEADER
I20240501 08:11:07.933707 31100 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:07.933806 31897 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:07.934950 30567 catalog_manager.cc:5484] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1a388864321846929919a37b570e8138 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:07.939530 30567 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1a388864321846929919a37b570e8138 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } }
W20240501 08:11:08.300693 31900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:08.301501 31900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:08.309459 31900 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:11:08.324330 31897 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.334411 31773 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.338281 31897 consensus_queue.cc:1035] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240501 08:11:08.390851 31897 consensus_queue.cc:1035] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.411152 31773 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.422782 31897 consensus_queue.cc:1035] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.423557 31897 consensus_queue.cc:1035] T e2393202bb014986b2496d3c99c65962 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.455053 31937 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.461745 31937 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.506080 31757 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.526088 31930 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:08.528179 31757 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 373.162 ms
time per row: 0.186581 ms
W20240501 08:11:09.393594 31165 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:11:09.720216 30562 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:11:09.729699 30950 tablet_service.cc:1467] Processing CreateTablet for tablet 9ff3c78386224499b7022dc9334ddd33 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.729810 31208 tablet_service.cc:1467] Processing CreateTablet for tablet 9ff3c78386224499b7022dc9334ddd33 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.730191 31208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.730196 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.731077 30948 tablet_service.cc:1467] Processing CreateTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.731379 30948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.735689 31340 tablet_service.cc:1467] Processing CreateTablet for tablet 9ff3c78386224499b7022dc9334ddd33 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.735704 30687 tablet_service.cc:1467] Processing CreateTablet for tablet 9ff3c78386224499b7022dc9334ddd33 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.735780 30689 tablet_service.cc:1467] Processing CreateTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.736152 31340 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.736196 30687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.736388 31079 tablet_service.cc:1467] Processing CreateTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.736763 31079 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.738898 31078 tablet_service.cc:1467] Processing CreateTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.739282 31078 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.739442 31339 tablet_service.cc:1467] Processing CreateTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.739778 31339 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.740376 30949 tablet_service.cc:1467] Processing CreateTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.740739 30949 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.741055 31080 tablet_service.cc:1467] Processing CreateTablet for tablet 9ff3c78386224499b7022dc9334ddd33 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.741086 30819 tablet_service.cc:1467] Processing CreateTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.741400 31080 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.741470 30819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.744750 30688 tablet_service.cc:1467] Processing CreateTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.744964 31209 tablet_service.cc:1467] Processing CreateTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.745118 30688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.745352 31209 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.746733 31210 tablet_service.cc:1467] Processing CreateTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:09.747128 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.748418 30689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:09.763514 31991 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:09.765905 31991 tablet_bootstrap.cc:654] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.766422 31986 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:09.768118 31986 tablet_bootstrap.cc:654] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.769523 31990 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:09.771138 31990 tablet_bootstrap.cc:654] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.773238 31992 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:09.774747 31992 tablet_bootstrap.cc:654] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.775578 31987 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:09.776211 31988 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:09.777671 31988 tablet_bootstrap.cc:654] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.778329 31987 tablet_bootstrap.cc:654] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.778824 31991 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:09.779165 31991 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.000s
I20240501 08:11:09.780339 31991 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.781088 31987 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:09.781286 31987 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240501 08:11:09.781498 31990 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:09.781703 31990 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.003s
I20240501 08:11:09.782038 31987 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.782338 31987 raft_consensus.cc:374] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.782472 31987 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.782528 31990 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.782810 31991 raft_consensus.cc:374] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.782948 31991 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.782824 31987 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.783262 31991 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.784060 31991 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:11:09.784202 31990 raft_consensus.cc:374] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.784353 31990 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.784828 31991 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:09.785055 31990 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.787060 31991 tablet_bootstrap.cc:654] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.787562 31990 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:11:09.788940 31992 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:09.789764 31986 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:09.790057 31986 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.024s user 0.003s sys 0.003s
I20240501 08:11:09.790980 31986 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.791291 31986 raft_consensus.cc:374] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.791429 31986 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.791525 31987 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20240501 08:11:09.791765 31986 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.792651 31986 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:11:09.792886 31992 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.020s user 0.008s sys 0.000s
I20240501 08:11:09.793354 31987 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:09.793845 31992 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.794975 31987 tablet_bootstrap.cc:654] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.795081 31992 raft_consensus.cc:374] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.795226 31992 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.795518 31992 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.796190 31992 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:09.796536 31992 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:09.798705 31992 tablet_bootstrap.cc:654] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.800010 31988 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:09.800233 31988 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.024s user 0.008s sys 0.011s
I20240501 08:11:09.801018 31988 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.801326 31988 raft_consensus.cc:374] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.801463 31988 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.801753 31988 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.802400 31988 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:09.803508 31988 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:09.804482 31893 raft_consensus.cc:482] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:09.804670 31893 raft_consensus.cc:504] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.805682 31893 leader_election.cc:288] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:09.805927 31987 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:09.806177 31987 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.005s
I20240501 08:11:09.806200 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:09.806399 31100 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:09.806501 31990 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:09.807148 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:09.807474 31987 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.807770 31987 raft_consensus.cc:374] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.807798 31988 tablet_bootstrap.cc:654] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.807880 31987 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.808081 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:09.808171 31987 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
W20240501 08:11:09.808435 30644 leader_election.cc:341] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:09.808584 31990 tablet_bootstrap.cc:654] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.808884 31987 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
W20240501 08:11:09.808914 30642 leader_election.cc:341] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:09.809247 31987 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:09.810058 31991 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:09.810225 31991 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.025s user 0.004s sys 0.000s
I20240501 08:11:09.810840 31987 tablet_bootstrap.cc:654] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.810947 31991 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.811197 31991 raft_consensus.cc:374] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.811275 31991 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.811506 31991 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.812256 31991 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:09.812542 31991 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:09.813436 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:09.814112 31991 tablet_bootstrap.cc:654] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
W20240501 08:11:09.814225 30643 leader_election.cc:341] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:09.814368 30643 leader_election.cc:302] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters: 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf
I20240501 08:11:09.814931 31893 raft_consensus.cc:2726] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:11:09.816489 31992 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:09.816761 31992 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.020s user 0.003s sys 0.001s
I20240501 08:11:09.818214 31992 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.818549 31992 raft_consensus.cc:374] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.818666 31992 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.818948 31992 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.820235 31992 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240501 08:11:09.820582 31992 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:09.822136 31992 tablet_bootstrap.cc:654] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.826210 31987 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:09.826458 31987 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.002s
I20240501 08:11:09.826848 31991 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:09.827072 31991 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.006s
I20240501 08:11:09.827172 31987 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.827416 31987 raft_consensus.cc:374] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.827512 31987 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.827608 31988 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:09.827800 31988 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.024s user 0.008s sys 0.001s
I20240501 08:11:09.827788 31987 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.828634 31988 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.828924 31988 raft_consensus.cc:374] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.829039 31988 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.829202 31987 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:09.829407 31988 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.830024 31988 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:11:09.830302 31992 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:09.830318 31988 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:09.830487 31992 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.002s
I20240501 08:11:09.831252 31992 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.831526 31992 raft_consensus.cc:374] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.831655 31992 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.831904 31992 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.832266 31988 tablet_bootstrap.cc:654] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:09.833201 31992 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:11:09.833915 31999 raft_consensus.cc:482] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:09.834105 31999 raft_consensus.cc:504] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.836287 31999 leader_election.cc:288] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:09.827850 31991 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.847898 31991 raft_consensus.cc:374] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.847939 31990 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:09.848054 31991 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.848750 31990 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.042s user 0.001s sys 0.020s
I20240501 08:11:09.848953 31991 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.850354 31991 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.023s user 0.002s sys 0.000s
I20240501 08:11:09.850955 31990 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.851203 31990 raft_consensus.cc:374] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.851277 31990 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.853122 31990 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.857218 31990 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240501 08:11:09.861644 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:09.862672 31988 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:09.862593 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:09.862813 31988 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.033s user 0.004s sys 0.005s
I20240501 08:11:09.862819 30836 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
W20240501 08:11:09.863152 30904 leader_election.cc:341] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:09.863494 31988 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.863665 31988 raft_consensus.cc:374] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:09.863749 31988 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:09.863986 31988 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.864600 31988 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:09.868127 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:09.868335 30707 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.868747 30903 leader_election.cc:302] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836; no voters: 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:11:09.869128 31999 raft_consensus.cc:2781] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:09.869248 31999 raft_consensus.cc:482] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:09.869346 31999 raft_consensus.cc:3037] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.869365 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:09.869589 31230 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.871632 31999 raft_consensus.cc:504] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.872638 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:09.872821 30707 raft_consensus.cc:3037] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.872794 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:09.872953 31999 leader_election.cc:288] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Requested vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:09.872972 30836 raft_consensus.cc:3037] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.874617 30707 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.875670 30836 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.876070 30906 leader_election.cc:302] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836; no voters:
I20240501 08:11:09.876336 31999 raft_consensus.cc:2781] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:09.876528 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:09.876722 31100 raft_consensus.cc:3037] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.876713 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:09.876961 31230 raft_consensus.cc:3037] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.879709 31100 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.879714 31230 raft_consensus.cc:2454] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.883886 31999 raft_consensus.cc:686] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Becoming Leader. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Running, Role: LEADER
I20240501 08:11:09.884384 32017 raft_consensus.cc:482] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:09.884505 31999 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:09.884548 32017 raft_consensus.cc:504] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.886018 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:09.886018 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:09.886224 31100 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.886214 31230 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.886288 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:09.886505 30707 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.886701 30904 leader_election.cc:302] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:09.887023 32018 raft_consensus.cc:2781] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:09.887069 32017 leader_election.cc:288] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:09.887142 32018 raft_consensus.cc:482] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:09.887274 32018 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.889768 30559 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57210dc0a07a4851a43be23a279eb836 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:09.892329 32018 raft_consensus.cc:504] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.893409 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:09.893622 31230 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.893802 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:09.893868 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:09.894009 31359 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:09.894045 31100 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.894259 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:09.894461 30707 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.896314 31230 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.896315 30707 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.896579 31100 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.896674 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:09.896845 31359 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:09.896787 30904 leader_election.cc:302] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:09.897064 31999 raft_consensus.cc:2781] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:09.896782 32018 leader_election.cc:288] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Requested vote from peers 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:09.897233 31999 raft_consensus.cc:686] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Becoming Leader. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Running, Role: LEADER
I20240501 08:11:09.897462 31999 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:09.898887 31359 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:09.900823 30559 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57210dc0a07a4851a43be23a279eb836 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:10.132201 31999 raft_consensus.cc:482] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:10.132409 31999 raft_consensus.cc:504] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:10.133445 31999 leader_election.cc:288] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:10.133868 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:10.134145 30707 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:10.134192 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:10.134416 31230 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:10.134831 30904 leader_election.cc:302] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:10.135114 31999 raft_consensus.cc:2781] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:10.135216 31999 raft_consensus.cc:482] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:10.135298 31999 raft_consensus.cc:3037] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:10.135385 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:10.135946 31359 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:10.136977 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:10.137225 31100 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:10.137951 31999 raft_consensus.cc:504] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:10.139261 31358 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:10.139477 31358 raft_consensus.cc:3037] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:10.139452 31230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:10.139650 31230 raft_consensus.cc:3037] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:10.140090 31999 leader_election.cc:288] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Requested vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:10.140933 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:10.141158 31100 raft_consensus.cc:3037] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:10.142230 31230 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:10.142246 31358 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:10.142652 30904 leader_election.cc:302] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:10.142969 31999 raft_consensus.cc:2781] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:10.143103 31999 raft_consensus.cc:686] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Becoming Leader. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Running, Role: LEADER
I20240501 08:11:10.143441 31100 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:10.143393 31999 consensus_queue.cc:237] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:11:10.144038 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:10.144366 30707 raft_consensus.cc:3037] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:10.146346 30707 raft_consensus.cc:2454] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:10.147541 30559 catalog_manager.cc:5484] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57210dc0a07a4851a43be23a279eb836 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:10.324190 31999 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.340162 31999 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.384091 31999 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.385022 31999 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.393082 32018 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.405937 32017 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.417006 31999 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.418133 32018 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:10.580893 32024 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:10.581230 32024 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:10.589053 32024 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240501 08:11:10.633169 32035 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.677451 32035 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.700277 32017 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:10.703126 32017 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 330.927 ms
time per row: 0.165463 ms
I20240501 08:11:11.061021 30561 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46462:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:11:11.069835 31079 tablet_service.cc:1467] Processing CreateTablet for tablet a465d27916734303bd9d1bed2a3dfb14 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.070099 31340 tablet_service.cc:1467] Processing CreateTablet for tablet a465d27916734303bd9d1bed2a3dfb14 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.070324 31079 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.070518 31340 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.070469 30819 tablet_service.cc:1467] Processing CreateTablet for tablet a465d27916734303bd9d1bed2a3dfb14 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.070848 30819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.072685 30689 tablet_service.cc:1467] Processing CreateTablet for tablet a465d27916734303bd9d1bed2a3dfb14 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.073117 30689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.073542 31209 tablet_service.cc:1467] Processing CreateTablet for tablet 3c02021c881348748fa99b0df2dfb228 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.073917 31209 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.076038 31210 tablet_service.cc:1467] Processing CreateTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.076403 31210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.076761 31078 tablet_service.cc:1467] Processing CreateTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.077132 31078 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.077896 30688 tablet_service.cc:1467] Processing CreateTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.078948 30950 tablet_service.cc:1467] Processing CreateTablet for tablet a465d27916734303bd9d1bed2a3dfb14 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.079335 30950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.080399 30687 tablet_service.cc:1467] Processing CreateTablet for tablet 3c02021c881348748fa99b0df2dfb228 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.082012 30820 tablet_service.cc:1467] Processing CreateTablet for tablet 3c02021c881348748fa99b0df2dfb228 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.082057 30818 tablet_service.cc:1467] Processing CreateTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.082419 30820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.082796 30818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.085340 31339 tablet_service.cc:1467] Processing CreateTablet for tablet 3c02021c881348748fa99b0df2dfb228 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.085755 31339 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.088346 31338 tablet_service.cc:1467] Processing CreateTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.088744 31338 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.093642 32105 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:11.094735 32106 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:11.095366 32105 tablet_bootstrap.cc:654] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.095489 30688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.096071 30687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.096649 32106 tablet_bootstrap.cc:654] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.098062 32104 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:11.098081 32105 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:11.098228 32105 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:11:11.098793 32106 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:11.098917 32106 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:11:11.098969 32105 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.099210 32105 raft_consensus.cc:374] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.099289 32105 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.099483 32105 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.099596 32106 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.099810 32106 raft_consensus.cc:374] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.099890 32106 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.100136 32106 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.100706 32106 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:11.100801 32105 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:11.100968 32106 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:11.101027 32105 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf: Bootstrap starting.
I20240501 08:11:11.102403 32106 tablet_bootstrap.cc:654] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.103052 32105 tablet_bootstrap.cc:654] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.107703 32104 tablet_bootstrap.cc:654] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.110174 30948 tablet_service.cc:1467] Processing CreateTablet for tablet 3c02021c881348748fa99b0df2dfb228 (DEFAULT_TABLE table=rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:11:11.110730 30948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:11.120723 32112 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:11.123900 32112 tablet_bootstrap.cc:654] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.133399 32106 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:11.133598 32106 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.033s user 0.017s sys 0.015s
I20240501 08:11:11.134407 32106 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.134712 32106 raft_consensus.cc:374] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.134788 32106 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.135071 32106 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.139462 32106 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.006s user 0.002s sys 0.001s
I20240501 08:11:11.140647 32106 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:11.142594 32106 tablet_bootstrap.cc:654] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.152529 32114 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:11.153368 32116 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:11.153602 32106 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: No bootstrap required, opened a new log
I20240501 08:11:11.153752 32106 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.010s
I20240501 08:11:11.154500 32106 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.154765 32106 raft_consensus.cc:374] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.154865 32106 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.155103 32116 tablet_bootstrap.cc:654] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.155170 32106 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.155728 32106 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:11.158125 32114 tablet_bootstrap.cc:654] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.160625 32112 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:11.160805 32112 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.040s user 0.012s sys 0.006s
I20240501 08:11:11.162600 32104 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:11.162752 32104 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.065s user 0.008s sys 0.016s
I20240501 08:11:11.163300 32112 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.163579 32112 raft_consensus.cc:374] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.163431 32104 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.163658 32104 raft_consensus.cc:374] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.163671 32112 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.163745 32104 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.163995 32104 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.164659 32104 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:11:11.164932 32104 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4: Bootstrap starting.
I20240501 08:11:11.165637 32116 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:11.165804 32116 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240501 08:11:11.166570 32116 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.166821 32116 raft_consensus.cc:374] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.166900 32116 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.167204 32112 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.167707 32104 tablet_bootstrap.cc:654] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.167893 32022 raft_consensus.cc:482] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:11.167979 32112 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
I20240501 08:11:11.168046 32022 raft_consensus.cc:504] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.168236 32112 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:11.169027 32022 leader_election.cc:288] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:11.169687 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:11.169822 32112 tablet_bootstrap.cc:654] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.169832 30837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:11.170053 30837 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 0.
W20240501 08:11:11.170284 31035 leader_election.cc:341] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:11.170859 32116 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.171176 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:11.171430 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
W20240501 08:11:11.172076 31033 leader_election.cc:341] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:11:11.172281 31033 leader_election.cc:341] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:11:11.172374 31033 leader_election.cc:302] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 1a388864321846929919a37b570e8138, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade
I20240501 08:11:11.172528 32116 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:11:11.172801 32022 raft_consensus.cc:2726] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:11:11.172834 32116 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:11.175786 32114 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:11.175983 32114 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.024s user 0.000s sys 0.003s
I20240501 08:11:11.176815 32114 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.177294 32114 raft_consensus.cc:374] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.177393 32114 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.177644 32114 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.177963 32104 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4: No bootstrap required, opened a new log
I20240501 08:11:11.178123 32104 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.002s
I20240501 08:11:11.178397 32114 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:11.178681 32114 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: Bootstrap starting.
I20240501 08:11:11.178843 32104 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.179076 32104 raft_consensus.cc:374] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.179147 32104 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.179414 32104 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.180099 32104 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:11:11.180351 32114 tablet_bootstrap.cc:654] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.182303 32105 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf: No bootstrap required, opened a new log
I20240501 08:11:11.182328 32116 tablet_bootstrap.cc:654] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.182472 32105 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf: Time spent bootstrapping tablet: real 0.081s user 0.032s sys 0.034s
I20240501 08:11:11.183172 32105 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.183473 32105 raft_consensus.cc:374] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.183532 32105 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9204a82012ac492dbc0da9c84c51dabf, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.183816 32105 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.184181 31985 raft_consensus.cc:482] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:11.184430 32105 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:11.184340 31985 raft_consensus.cc:504] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.185284 31985 leader_election.cc:288] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:11.185616 31229 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:11.185822 31229 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:11.184065 32112 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:11.185925 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:11.186111 32112 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.000s
I20240501 08:11:11.186162 31359 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:11.186151 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836" is_pre_election: true
I20240501 08:11:11.186368 30967 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:11.186524 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:11.186739 30707 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 0.
I20240501 08:11:11.186818 30773 leader_election.cc:302] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade; no voters:
I20240501 08:11:11.187129 32112 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.187388 32112 raft_consensus.cc:374] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.187466 32112 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.187737 32112 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.188148 31985 raft_consensus.cc:2781] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:11.188271 31985 raft_consensus.cc:482] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:11.188367 31985 raft_consensus.cc:3037] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.188565 32114 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: No bootstrap required, opened a new log
I20240501 08:11:11.188728 32114 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20240501 08:11:11.189394 32114 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.189633 32114 raft_consensus.cc:374] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.189709 32114 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.189924 32114 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.190603 32112 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:11:11.191059 32112 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Bootstrap starting.
I20240501 08:11:11.190936 31985 raft_consensus.cc:504] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.191814 31985 leader_election.cc:288] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Requested vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:11.192047 31229 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:11.192102 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:11.192240 31229 raft_consensus.cc:3037] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.192320 31359 raft_consensus.cc:3037] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.192878 32112 tablet_bootstrap.cc:654] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.194373 32114 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240501 08:11:11.194639 32116 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:11.194855 32116 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.022s user 0.004s sys 0.000s
I20240501 08:11:11.194970 31229 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:11.194977 31359 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:11.195304 32112 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: No bootstrap required, opened a new log
I20240501 08:11:11.195438 32112 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:11:11.196177 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:11.196182 30967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c02021c881348748fa99b0df2dfb228" candidate_uuid: "1a388864321846929919a37b570e8138" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:11.196357 30707 raft_consensus.cc:3037] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.196385 30967 raft_consensus.cc:3037] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.196405 32112 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.196614 32112 raft_consensus.cc:374] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.196702 32112 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f37f9ed957342dba1271066c1bbeade, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.196923 32112 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.197430 32112 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:11:11.195840 32116 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.197811 32116 raft_consensus.cc:374] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.197903 32116 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.198141 32116 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.198408 30967 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:11.198783 32116 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:11:11.198945 30707 raft_consensus.cc:2454] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a388864321846929919a37b570e8138 in term 1.
I20240501 08:11:11.198987 30773 leader_election.cc:302] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade; no voters:
I20240501 08:11:11.199072 32116 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:11.199321 31985 raft_consensus.cc:2781] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:11.200479 31985 raft_consensus.cc:686] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Becoming Leader. State: Replica: 1a388864321846929919a37b570e8138, State: Running, Role: LEADER
I20240501 08:11:11.200651 32116 tablet_bootstrap.cc:654] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Neither blocks nor log segments found. Creating new log.
I20240501 08:11:11.200798 31985 consensus_queue.cc:237] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.202411 32116 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: No bootstrap required, opened a new log
I20240501 08:11:11.202553 32116 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:11:11.203797 32116 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.204047 32116 raft_consensus.cc:374] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:11:11.204135 32116 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: FOLLOWER
I20240501 08:11:11.204363 32116 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.204821 32116 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:11:11.205169 31999 raft_consensus.cc:482] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:11.205320 31999 raft_consensus.cc:504] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.206090 31893 raft_consensus.cc:482] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:11:11.206187 31999 leader_election.cc:288] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:11:11.206233 31893 raft_consensus.cc:504] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.206374 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:11.206516 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" is_pre_election: true
I20240501 08:11:11.206580 31359 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:11.206727 30707 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:11.207054 30837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:11.207273 30837 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:11.207274 30905 leader_election.cc:302] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 6f37f9ed957342dba1271066c1bbeade; no voters:
I20240501 08:11:11.207325 31893 leader_election.cc:288] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:11.207563 31999 raft_consensus.cc:2781] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:11.207679 31999 raft_consensus.cc:482] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:11.207629 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:11.207772 31999 raft_consensus.cc:3037] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.207844 31100 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 0.
I20240501 08:11:11.205706 30561 catalog_manager.cc:5484] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 reported cstate change: term changed from 0 to 1, leader changed from <none> to 1a388864321846929919a37b570e8138 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:11.208325 31098 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:11:11.208550 31098 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:11.209002 31229 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:11:11.209239 31229 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:11.210209 30643 leader_election.cc:302] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:11.210536 31999 raft_consensus.cc:504] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.211700 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:11.211767 30707 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:11.211884 31100 raft_consensus.cc:3037] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.211980 30707 raft_consensus.cc:3037] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.212108 31893 raft_consensus.cc:2781] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:11:11.212211 31893 raft_consensus.cc:482] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:11:11.212293 31893 raft_consensus.cc:3037] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.212594 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade" is_pre_election: true
I20240501 08:11:11.212821 31359 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 0.
I20240501 08:11:11.214392 31100 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:11.214716 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:11.214910 31360 raft_consensus.cc:3037] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.214903 30707 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:11.214876 31893 raft_consensus.cc:504] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.215421 30903 leader_election.cc:302] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:11.215727 32017 raft_consensus.cc:2781] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:11.215731 31893 leader_election.cc:288] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Requested vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:11.215849 32017 raft_consensus.cc:686] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Becoming Leader. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Running, Role: LEADER
I20240501 08:11:11.216187 32017 consensus_queue.cc:237] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } }
I20240501 08:11:11.216459 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:11.216645 31359 raft_consensus.cc:3037] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.218250 31999 leader_election.cc:288] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [CANDIDATE]: Term 1 election: Requested vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:11:11.218986 31360 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:11.219215 31359 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
I20240501 08:11:11.219950 31229 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:11.220204 31229 raft_consensus.cc:3037] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.220168 30837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a465d27916734303bd9d1bed2a3dfb14" candidate_uuid: "57210dc0a07a4851a43be23a279eb836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:11.220384 30837 raft_consensus.cc:3037] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.223078 30837 raft_consensus.cc:2454] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57210dc0a07a4851a43be23a279eb836 in term 1.
I20240501 08:11:11.223078 31229 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
I20240501 08:11:11.223533 30643 leader_election.cc:302] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf; no voters:
I20240501 08:11:11.223918 31893 raft_consensus.cc:2781] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:11:11.224124 31100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:11:11.224370 31100 raft_consensus.cc:3037] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.227425 31100 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
I20240501 08:11:11.229990 31893 raft_consensus.cc:686] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Becoming Leader. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Running, Role: LEADER
I20240501 08:11:11.230361 31893 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } }
I20240501 08:11:11.230756 30561 catalog_manager.cc:5484] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: term changed from 0 to 1, leader changed from <none> to 57210dc0a07a4851a43be23a279eb836 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } }
I20240501 08:11:11.234623 30836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:11:11.234858 30836 raft_consensus.cc:3037] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:11:11.235411 30837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a388864321846929919a37b570e8138" is_pre_election: true
I20240501 08:11:11.235452 30567 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:11.237373 30836 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3170be3b2bbb4b3bbd8a37ac5f91de89 in term 1.
W20240501 08:11:11.585229 32133 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:11.585654 32133 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:11.593658 32133 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:11:11.611845 31985 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
I20240501 08:11:11.633466 32017 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.663394 31985 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.669937 31893 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.670643 32017 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.684906 31985 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.688180 31985 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.709921 32017 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.722326 32017 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.728027 31893 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:11:11.744566 31165 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240501 08:11:11.782397 32131 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:11:11.814646 32184 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
INSERT report
rows total: 2000
time total: 402.54 ms
time per row: 0.20127 ms
W20240501 08:11:12.121102 31165 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.135:36533: connect: Connection refused (error 111) [suppressed 10 similar messages]
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:11:13.032812 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.135:36533
--local_ip_for_outbound_sockets=127.28.1.135
--tserver_master_addrs=127.28.1.190:35983
--webserver_port=42973
--webserver_interface=127.28.1.135
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:11:13.365856 32213 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:11:13.366217 32213 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:11:13.366374 32213 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:11:13.374315 32213 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:11:13.374455 32213 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:11:13.374576 32213 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.135
I20240501 08:11:13.383644 32213 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.135:36533
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.28.1.135
--webserver_port=42973
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.135
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:11:13.385022 32213 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:11:13.387100 32213 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:11:13.399941 32224 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:13.400329 32221 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:11:13.400918 32222 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:11:13.401616 32213 server_base.cc:1034] running on GCE node
I20240501 08:11:13.402226 32213 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:11:13.403219 32213 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:11:13.404435 32213 hybrid_clock.cc:648] HybridClock initialized: now 1714551073404383 us; error 40 us; skew 500 ppm
I20240501 08:11:13.404786 32213 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:11:13.407189 32213 webserver.cc:460] Webserver started at http://127.28.1.135:42973/ using document root <none> and password file <none>
I20240501 08:11:13.407928 32213 fs_manager.cc:359] Metadata directory not provided
I20240501 08:11:13.408090 32213 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:11:13.413830 32213 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.004s sys 0.000s
I20240501 08:11:13.416707 32230 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:11:13.418135 32213 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:11:13.418296 32213 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
format_stamp: "Formatted at 2024-05-01 08:11:01 on dist-test-slave-jj4n"
I20240501 08:11:13.419162 32213 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:11:13.447347 32213 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:11:13.448391 32213 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:11:13.448719 32213 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:11:13.449895 32213 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:11:13.452409 32237 ts_tablet_manager.cc:522] Loading tablet metadata (0/1 complete)
I20240501 08:11:13.457675 32213 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240501 08:11:13.457798 32213 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.006s user 0.000s sys 0.001s
I20240501 08:11:13.457919 32213 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240501 08:11:13.461051 32213 ts_tablet_manager.cc:590] Registered 1 tablets
I20240501 08:11:13.461165 32213 ts_tablet_manager.cc:569] Time spent register tablets: real 0.003s user 0.001s sys 0.002s
I20240501 08:11:13.461508 32237 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:13.480175 32237 log.cc:824] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Log is configured to *not* fsync() on all Append() calls
I20240501 08:11:13.483160 32237 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:13.484036 32237 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:13.485874 32237 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 0.024s user 0.007s sys 0.015s
I20240501 08:11:13.494319 32237 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:13.495150 32237 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: FOLLOWER
I20240501 08:11:13.496078 32237 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } }
I20240501 08:11:13.497946 32237 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.012s user 0.011s sys 0.000s
I20240501 08:11:13.511608 32213 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.135:36533
I20240501 08:11:13.511660 32345 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.135:36533 every 8 connection(s)
I20240501 08:11:13.512959 32213 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
W20240501 08:11:13.514068 32347 tablet.cc:2369] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:11:13.520324 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 32213
I20240501 08:11:13.520593 28678 test_util.cc:268] Using random seed: 128957373
I20240501 08:11:13.529983 32346 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:11:13.530298 32346 heartbeater.cc:464] Registering TS with master...
I20240501 08:11:13.531168 32346 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:11:13.533915 30566 ts_manager.cc:194] Re-registered known tserver with Master: fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533)
I20240501 08:11:13.535171 30566 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.135:51113
I20240501 08:11:13.542562 28678 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_0 already exists
I20240501 08:11:13.543011 28678 test_util.cc:268] Using random seed: 128979789
I20240501 08:11:13.563436 28678 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_1 already exists
I20240501 08:11:13.563982 28678 test_util.cc:268] Using random seed: 129000735
I20240501 08:11:13.611939 28678 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_2 already exists
I20240501 08:11:13.659739 28678 test_util.cc:268] Using random seed: 129096492
I20240501 08:11:13.684111 32346 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
I20240501 08:11:13.781738 28678 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_3 already exists
I20240501 08:11:13.832583 28678 test_util.cc:268] Using random seed: 129269348
W20240501 08:11:13.875471 31294 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 44657.4us
W20240501 08:11:14.050256 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 80259.8us
W20240501 08:11:14.050384 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 73734.7us
W20240501 08:11:14.047135 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 77045.5us
I20240501 08:11:14.098050 28678 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_4 already exists
W20240501 08:11:14.321283 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 55095.6us
W20240501 08:11:15.122352 32532 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.551s user 0.010s sys 0.007s
W20240501 08:11:15.122567 32532 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.551s user 0.010s sys 0.007s
W20240501 08:11:15.422132 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 56535.6us
W20240501 08:11:15.502743 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 78633.1us
W20240501 08:11:15.793089 32509 thread.cc:636] raft [worker] (thread pool) Time spent creating pthread: real 0.507s user 0.000s sys 0.010s
W20240501 08:11:15.794895 32509 thread.cc:602] raft [worker] (thread pool) Time spent starting thread: real 0.508s user 0.000s sys 0.010s
W20240501 08:11:15.892879 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 48319.8us
W20240501 08:11:15.893630 31164 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 48374us
W20240501 08:11:15.944336 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 103979us
W20240501 08:11:16.268626 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 46615.2us
W20240501 08:11:16.774741 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 73070.7us
W20240501 08:11:17.796839 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 48310.1us
W20240501 08:11:17.893064 30775 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 43744.8us
I20240501 08:11:17.951689 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240501 08:11:17.969759 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 54226.4us
I20240501 08:11:18.191444 30948 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:11:18.208138 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:18.273670 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:11:18.409068 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240501 08:11:18.596239 31094 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.646s user 0.002s sys 0.008s
W20240501 08:11:18.596877 31094 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.647s user 0.003s sys 0.008s
I20240501 08:11:18.643119 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240501 08:11:18.715473 31035 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 290476us
I20240501 08:11:18.775563 31079 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240501 08:11:19.338506 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 64244.4us
W20240501 08:11:19.446034 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 157800us
W20240501 08:11:19.500135 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 99716.9us
W20240501 08:11:19.624807 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 53118.1us
W20240501 08:11:19.698942 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 48071.6us
W20240501 08:11:19.705027 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 73948.6us
W20240501 08:11:20.521683 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 74835.4us
W20240501 08:11:20.851713 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 61944.5us
W20240501 08:11:21.144969 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 45115.3us
W20240501 08:11:22.391288 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 177957us
W20240501 08:11:22.629820 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 46554us
I20240501 08:11:22.944229 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:22.953145 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:11:23.012198 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:11:23.026904 30948 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
W20240501 08:11:23.205030 31293 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 51870.4us
I20240501 08:11:23.284179 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:11:23.298597 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:23.315383 31079 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240501 08:11:23.528477 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 106097us
W20240501 08:11:23.868289 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 63040.8us
W20240501 08:11:24.043470 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 95712.7us
W20240501 08:11:24.486896 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 45303.1us
W20240501 08:11:24.487668 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 43736.2us
I20240501 08:11:24.593712 30966 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 100, Committed index: 100, Last appended: 1.100, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:24.600564 31100 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240501 08:11:24.601725 32566 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
I20240501 08:11:24.606027 30709 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240501 08:11:24.607641 32566 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
W20240501 08:11:24.611249 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 46911.7us
I20240501 08:11:24.613137 32566 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.101: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:24.616194 30709 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:24.653960 30566 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:24.655818 31098 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:24.665866 30840 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240501 08:11:24.676999 32675 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 99, Time since last communication: 0.000s
I20240501 08:11:24.698472 31228 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
W20240501 08:11:24.723663 30905 consensus_peers.cc:485] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8302dd0a0d2e4a20bdc00e27da939e8a. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:24.742847 30835 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:24.758109 32712 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.041s
W20240501 08:11:25.000126 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 69095.4us
W20240501 08:11:25.003684 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 74239.1us
I20240501 08:11:25.006481 31229 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index -1 to 101, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.253861 373 ts_tablet_manager.cc:902] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:25.296662 31230 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 89, Committed index: 89, Last appended: 1.89, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:25.302652 31356 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 90. (index mismatch)
W20240501 08:11:25.306982 31165 consensus_peers.cc:485] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3b8a23ca81da4039b7c18c84b2f2d8fb. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:25.310369 32422 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 90, Last known committed idx: 88, Time since last communication: 0.000s
I20240501 08:11:25.312723 30967 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 90. (index mismatch)
I20240501 08:11:25.313298 32422 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 90, Last known committed idx: 89, Time since last communication: 0.000s
I20240501 08:11:25.313635 30706 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 90. (index mismatch)
I20240501 08:11:25.322611 333 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.90: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.329124 32422 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 90, Last known committed idx: 89, Time since last communication: 0.000s
I20240501 08:11:25.331434 30969 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.90: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.338522 30559 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:25.340080 31355 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.90: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.351948 30706 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.90: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.406975 30834 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 90. (index mismatch)
I20240501 08:11:25.445587 333 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 90, Last known committed idx: 89, Time since last communication: 0.000s
I20240501 08:11:25.452840 30837 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.90: config changed from index -1 to 90, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:25.693028 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 63997.3us
I20240501 08:11:25.733870 373 tablet_copy_client.cc:308] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:39451
I20240501 08:11:25.755234 31357 consensus_queue.cc:237] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 106, Committed index: 106, Last appended: 1.106, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:25.818148 30837 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
I20240501 08:11:25.818871 375 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 106, Time since last communication: 0.000s
I20240501 08:11:25.821148 31098 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
I20240501 08:11:25.821861 375 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 106, Time since last communication: 0.000s
I20240501 08:11:25.823990 31229 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
W20240501 08:11:25.821858 31295 consensus_peers.cc:485] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 120d5a7fb6704b4eb07c02ab8dabf6cd. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:25.824894 378 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 LEADER]: Committing config change with OpId 1.107: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.825789 30967 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 106. Preceding OpId from leader: term: 1 index: 107. (index mismatch)
I20240501 08:11:25.827507 32548 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 106, Time since last communication: 0.000s
I20240501 08:11:25.829020 30967 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.831948 31098 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.832563 30559 catalog_manager.cc:5484] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade reported cstate change: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "6f37f9ed957342dba1271066c1bbeade" committed_config { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:25.832031 30835 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.835511 378 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 107, Last known committed idx: 106, Time since last communication: 0.000s
W20240501 08:11:25.831184 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 139720us
W20240501 08:11:25.870497 31033 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 106487us
I20240501 08:11:25.877185 31225 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.107: config changed from index -1 to 107, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:25.886662 30990 tablet_copy_service.cc:140] P 57210dc0a07a4851a43be23a279eb836: Received BeginTabletCopySession request for tablet 8302dd0a0d2e4a20bdc00e27da939e8a from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:57155)
I20240501 08:11:25.886996 30990 tablet_copy_service.cc:161] P 57210dc0a07a4851a43be23a279eb836: Beginning new tablet copy session on tablet 8302dd0a0d2e4a20bdc00e27da939e8a from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:57155: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:11:25.896489 30990 tablet_copy_source_session.cc:205] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Tablet Copy: opened 0 blocks and 1 log segments
W20240501 08:11:25.900574 31035 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 52176.9us
I20240501 08:11:25.901654 373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
W20240501 08:11:25.942615 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 43693.1us
I20240501 08:11:25.993235 373 tablet_copy_client.cc:783] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:25.995028 373 tablet_copy_client.cc:647] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
W20240501 08:11:26.070703 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 80868.8us
I20240501 08:11:26.115873 30970 consensus_queue.cc:237] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 132, Committed index: 132, Last appended: 1.132, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } }
I20240501 08:11:26.119092 31357 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 132. Preceding OpId from leader: term: 1 index: 133. (index mismatch)
I20240501 08:11:26.120497 320 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 133, Last known committed idx: 132, Time since last communication: 0.000s
I20240501 08:11:26.128329 30708 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 132. Preceding OpId from leader: term: 1 index: 133. (index mismatch)
I20240501 08:11:26.128937 32566 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 133, Last known committed idx: 132, Time since last communication: 0.000s
W20240501 08:11:26.131832 30906 consensus_peers.cc:485] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9c025c9ec2c74a54ab24c40ed11639f6. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:26.132719 32712 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.133: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New config: { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } } }
I20240501 08:11:26.141896 30563 catalog_manager.cc:5484] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:26.148505 31357 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.133: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New config: { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } } }
I20240501 08:11:26.151008 30706 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.133: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New config: { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } } }
I20240501 08:11:26.202906 417 ts_tablet_manager.cc:902] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:26.225664 31228 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 130. Preceding OpId from leader: term: 1 index: 133. (index mismatch)
I20240501 08:11:26.227015 373 tablet_copy_client.cc:523] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:26.231541 31095 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 130. Preceding OpId from leader: term: 1 index: 133. (index mismatch)
I20240501 08:11:26.234558 373 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:26.309202 320 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 133, Last known committed idx: 129, Time since last communication: 0.000s
I20240501 08:11:26.310585 320 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 133, Last known committed idx: 129, Time since last communication: 0.000s
I20240501 08:11:26.318621 31230 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.133: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New config: { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } } }
I20240501 08:11:26.330262 31098 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.133: config changed from index -1 to 133, NON_VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) added. New config: { opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } } }
I20240501 08:11:26.345623 31225 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 127, Committed index: 127, Last appended: 1.128, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } }
I20240501 08:11:26.351066 30834 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240501 08:11:26.351837 333 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 127, Time since last communication: 0.000s
W20240501 08:11:26.355257 31163 consensus_peers.cc:485] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5730faa05092410da14e00e24ad74c86. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:26.355350 30968 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 126. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240501 08:11:26.363389 420 ts_tablet_manager.cc:902] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579)
I20240501 08:11:26.368590 31100 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240501 08:11:26.368870 333 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 126, Time since last communication: 0.000s
I20240501 08:11:26.369166 32422 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 127, Time since last communication: 0.000s
I20240501 08:11:26.372331 31357 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 125. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240501 08:11:26.373766 32422 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.129: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.379995 337 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 125, Time since last communication: 0.000s
I20240501 08:11:26.384794 31100 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.385320 30834 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.392498 30565 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.395761 30967 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.406059 31358 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index -1 to 129, NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) added. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } } }
I20240501 08:11:26.412842 417 tablet_copy_client.cc:308] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
W20240501 08:11:26.474694 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 94043.2us
I20240501 08:11:26.563912 420 tablet_copy_client.cc:308] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:41579
I20240501 08:11:26.628261 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:35117)
I20240501 08:11:26.628546 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet 3b8a23ca81da4039b7c18c84b2f2d8fb from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:35117: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:11:26.646401 31250 tablet_copy_source_session.cc:205] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:26.660266 417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:26.672214 417 tablet_copy_client.cc:783] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:26.672575 417 tablet_copy_client.cc:647] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:26.688594 417 tablet_copy_client.cc:523] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:26.714212 417 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
W20240501 08:11:26.726555 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 48268.5us
I20240501 08:11:26.776600 446 ts_tablet_manager.cc:902] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:26.789588 445 ts_tablet_manager.cc:902] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:26.751379 31098 consensus_queue.cc:237] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 109, Committed index: 109, Last appended: 1.109, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:26.891331 445 tablet_copy_client.cc:308] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:39451
I20240501 08:11:26.893329 30990 tablet_copy_service.cc:140] P 57210dc0a07a4851a43be23a279eb836: Received BeginTabletCopySession request for tablet 9c025c9ec2c74a54ab24c40ed11639f6 from peer 1a388864321846929919a37b570e8138 ({username='slave'} at 127.28.1.130:49241)
I20240501 08:11:26.893541 30990 tablet_copy_service.cc:161] P 57210dc0a07a4851a43be23a279eb836: Beginning new tablet copy session on tablet 9c025c9ec2c74a54ab24c40ed11639f6 from peer 1a388864321846929919a37b570e8138 at {username='slave'} at 127.28.1.130:49241: session id = 1a388864321846929919a37b570e8138-9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:11:26.899143 30990 tablet_copy_source_session.cc:205] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:26.904383 445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9c025c9ec2c74a54ab24c40ed11639f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:26.928627 445 tablet_copy_client.cc:783] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:26.929024 445 tablet_copy_client.cc:647] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:26.951435 445 tablet_copy_client.cc:523] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:26.951930 446 tablet_copy_client.cc:308] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:11:26.960619 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet 5730faa05092410da14e00e24ad74c86 from peer 3170be3b2bbb4b3bbd8a37ac5f91de89 ({username='slave'} at 127.28.1.129:57131)
I20240501 08:11:26.960812 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet 5730faa05092410da14e00e24ad74c86 from peer 3170be3b2bbb4b3bbd8a37ac5f91de89 at {username='slave'} at 127.28.1.129:57131: session id = 3170be3b2bbb4b3bbd8a37ac5f91de89-5730faa05092410da14e00e24ad74c86
I20240501 08:11:26.963855 31250 tablet_copy_source_session.cc:205] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:26.990221 445 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Bootstrap starting.
I20240501 08:11:27.043869 446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:27.056474 446 tablet_copy_client.cc:783] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:27.056928 446 tablet_copy_client.cc:647] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:27.121573 30967 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 109. Preceding OpId from leader: term: 1 index: 111. (index mismatch)
I20240501 08:11:27.122263 398 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 110, Last known committed idx: 109, Time since last communication: 0.000s
I20240501 08:11:27.135669 446 tablet_copy_client.cc:523] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:27.145098 446 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap starting.
I20240501 08:11:27.146090 30706 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 109. Preceding OpId from leader: term: 1 index: 111. (index mismatch)
I20240501 08:11:27.146757 398 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 110, Last known committed idx: 109, Time since last communication: 0.000s
I20240501 08:11:27.153189 394 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 LEADER]: Committing config change with OpId 1.110: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.160600 30708 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.110: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.168921 30563 catalog_manager.cc:5484] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.172045 31380 tablet_copy_service.cc:140] P 6f37f9ed957342dba1271066c1bbeade: Received BeginTabletCopySession request for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:56725)
I20240501 08:11:27.172333 31380 tablet_copy_service.cc:161] P 6f37f9ed957342dba1271066c1bbeade: Beginning new tablet copy session on tablet 120d5a7fb6704b4eb07c02ab8dabf6cd from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:56725: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:11:27.176254 31380 tablet_copy_source_session.cc:205] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:27.176085 30969 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.110: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.193732 30834 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 109. Preceding OpId from leader: term: 1 index: 111. (index mismatch)
I20240501 08:11:27.195672 394 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 110, Last known committed idx: 109, Time since last communication: 0.000s
I20240501 08:11:27.205751 420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:27.212672 30834 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.110: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.213193 420 tablet_copy_client.cc:783] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:27.213635 420 tablet_copy_client.cc:647] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:27.314118 31357 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 109. Preceding OpId from leader: term: 1 index: 111. (index mismatch)
I20240501 08:11:27.318868 420 tablet_copy_client.cc:523] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:27.329504 394 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 110, Last known committed idx: 109, Time since last communication: 0.000s
I20240501 08:11:27.350154 31359 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.110: config changed from index -1 to 110, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.416570 420 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
W20240501 08:11:27.497591 31035 consensus_peers.cc:485] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1280b2f88bbc470bbc9bc49c95a4b108. This is attempt 1: this message will repeat every 5th retry.
W20240501 08:11:27.556005 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 62063.6us
I20240501 08:11:27.746894 480 ts_tablet_manager.cc:902] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:11:27.785351 31228 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 113, Committed index: 113, Last appended: 1.114, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:27.788057 31356 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
W20240501 08:11:27.788393 31165 consensus_peers.cc:485] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bc20191679084ecfbd360e23d68abd62. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:27.788775 391 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 113, Time since last communication: 0.000s
I20240501 08:11:27.792913 30834 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240501 08:11:27.793905 391 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 113, Time since last communication: 0.000s
I20240501 08:11:27.802011 31095 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 113. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240501 08:11:27.802505 391 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 113, Time since last communication: 0.000s
I20240501 08:11:27.809177 337 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.115: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.810189 31098 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.820248 31359 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.822680 30563 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:27.833180 30706 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240501 08:11:27.841205 32494 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 113, Time since last communication: 0.000s
I20240501 08:11:27.942452 30838 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:27.982604 30704 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index -1 to 115, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:28.209501 420 log.cc:924] Time spent T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.088s user 0.000s sys 0.000s
I20240501 08:11:28.309193 480 tablet_copy_client.cc:308] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:41135
I20240501 08:11:28.354393 516 ts_tablet_manager.cc:902] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
W20240501 08:11:28.391383 30905 consensus_peers.cc:485] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240501 08:11:28.422559 417 log.cc:924] Time spent T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.052s user 0.002s sys 0.000s
W20240501 08:11:28.460536 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 55956.5us
I20240501 08:11:28.501866 31120 tablet_copy_service.cc:140] P 9fc0640dbc0d4208bfb536949d1bb0c4: Received BeginTabletCopySession request for tablet 1280b2f88bbc470bbc9bc49c95a4b108 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:34317)
I20240501 08:11:28.502143 31120 tablet_copy_service.cc:161] P 9fc0640dbc0d4208bfb536949d1bb0c4: Beginning new tablet copy session on tablet 1280b2f88bbc470bbc9bc49c95a4b108 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:34317: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-1280b2f88bbc470bbc9bc49c95a4b108
I20240501 08:11:28.508850 31120 tablet_copy_source_session.cc:205] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:28.510939 480 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1280b2f88bbc470bbc9bc49c95a4b108. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:28.519265 480 tablet_copy_client.cc:783] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:28.519690 480 tablet_copy_client.cc:647] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:28.532125 480 tablet_copy_client.cc:523] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:28.539001 480 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
W20240501 08:11:28.615666 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 46335us
W20240501 08:11:28.729358 417 log.cc:924] Time spent T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.221s user 0.004s sys 0.000s
I20240501 08:11:28.862135 516 tablet_copy_client.cc:308] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:11:28.862919 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet bc20191679084ecfbd360e23d68abd62 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:35117)
I20240501 08:11:28.863134 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet bc20191679084ecfbd360e23d68abd62 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:35117: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-bc20191679084ecfbd360e23d68abd62
I20240501 08:11:28.865808 31250 tablet_copy_source_session.cc:205] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:28.867862 516 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:29.026508 516 tablet_copy_client.cc:783] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:29.027264 516 tablet_copy_client.cc:647] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:29.032557 516 tablet_copy_client.cc:523] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
W20240501 08:11:29.077591 31163 consensus_peers.cc:485] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240501 08:11:29.107421 420 log.cc:924] Time spent T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.160s user 0.004s sys 0.000s
I20240501 08:11:29.131639 516 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:29.208146 417 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=98 overwritten=0 applied=98 ignored=0} inserts{seen=729 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
W20240501 08:11:29.211944 446 log.cc:924] Time spent T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Append to log took a long time: real 0.172s user 0.003s sys 0.000s
I20240501 08:11:29.212239 417 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:29.212792 417 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 2.498s user 0.152s sys 0.020s
I20240501 08:11:29.219908 417 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.220291 417 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:29.220538 417 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 98, Last appended: 1.98, Last appended by leader: 98, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 90 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.247817 417 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.035s user 0.008s sys 0.000s
I20240501 08:11:29.249928 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-3b8a23ca81da4039b7c18c84b2f2d8fb received from {username='slave'} at 127.28.1.135:35117
I20240501 08:11:29.250130 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-3b8a23ca81da4039b7c18c84b2f2d8fb on tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with peer fe99ab4507a24af4ba7fa5b5278c5ac6
W20240501 08:11:29.318203 30906 consensus_peers.cc:485] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240501 08:11:29.538945 420 log.cc:924] Time spent T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.095s user 0.001s sys 0.000s
W20240501 08:11:29.595140 446 log.cc:924] Time spent T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Append to log took a long time: real 0.099s user 0.004s sys 0.000s
I20240501 08:11:29.686547 32301 raft_consensus.cc:1203] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.89->[1.90-1.118] Dedup: 1.98->[1.99-1.118]
I20240501 08:11:29.705906 373 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=122 overwritten=0 applied=122 ignored=0} inserts{seen=794 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:29.710657 373 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:29.711937 373 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 3.477s user 0.195s sys 0.022s
I20240501 08:11:29.722112 373 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.745303 373 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:29.745649 373 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 122, Last appended: 1.122, Last appended by leader: 122, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.746493 373 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.032s user 0.000s sys 0.002s
I20240501 08:11:29.771648 30990 tablet_copy_service.cc:342] P 57210dc0a07a4851a43be23a279eb836: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-8302dd0a0d2e4a20bdc00e27da939e8a received from {username='slave'} at 127.28.1.135:57155
I20240501 08:11:29.771870 30990 tablet_copy_service.cc:433] P 57210dc0a07a4851a43be23a279eb836: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-8302dd0a0d2e4a20bdc00e27da939e8a on tablet 8302dd0a0d2e4a20bdc00e27da939e8a with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:29.823905 495 raft_consensus.cc:1050] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:29.825196 495 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 119, Committed index: 119, Last appended: 1.119, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:29.832617 32300 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 119. Preceding OpId from leader: term: 1 index: 121. (index mismatch)
I20240501 08:11:29.833353 494 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 120, Last known committed idx: 119, Time since last communication: 0.000s
I20240501 08:11:29.840935 30968 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 119. Preceding OpId from leader: term: 1 index: 121. (index mismatch)
I20240501 08:11:29.842188 30837 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 119. Preceding OpId from leader: term: 1 index: 121. (index mismatch)
I20240501 08:11:29.845957 32494 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 120, Last known committed idx: 119, Time since last communication: 0.000s
I20240501 08:11:29.846873 32422 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 120, Last known committed idx: 119, Time since last communication: 0.000s
I20240501 08:11:29.880343 480 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=122 overwritten=0 applied=122 ignored=0} inserts{seen=807 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:29.882045 32494 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:29.883302 516 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=121 overwritten=0 applied=121 ignored=0} inserts{seen=752 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:29.887748 516 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:29.888154 516 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 0.757s user 0.155s sys 0.022s
I20240501 08:11:29.888949 516 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.889360 516 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:29.889597 516 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 121, Last appended: 1.121, Last appended by leader: 121, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.893940 516 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.006s user 0.000s sys 0.001s
I20240501 08:11:29.894910 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-bc20191679084ecfbd360e23d68abd62 received from {username='slave'} at 127.28.1.135:35117
I20240501 08:11:29.895063 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-bc20191679084ecfbd360e23d68abd62 on tablet bc20191679084ecfbd360e23d68abd62 with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:29.887269 480 tablet_bootstrap.cc:492] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:29.897425 480 ts_tablet_manager.cc:1367] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 1.359s user 0.188s sys 0.003s
I20240501 08:11:29.900792 480 raft_consensus.cc:348] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.901103 480 raft_consensus.cc:729] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:29.905056 480 consensus_queue.cc:260] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 122, Last appended: 1.122, Last appended by leader: 122, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 110 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:29.916123 480 ts_tablet_manager.cc:1398] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.019s user 0.003s sys 0.000s
I20240501 08:11:29.916939 31120 tablet_copy_service.cc:342] P 9fc0640dbc0d4208bfb536949d1bb0c4: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-1280b2f88bbc470bbc9bc49c95a4b108 received from {username='slave'} at 127.28.1.135:34317
I20240501 08:11:29.917116 31120 tablet_copy_service.cc:433] P 9fc0640dbc0d4208bfb536949d1bb0c4: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-1280b2f88bbc470bbc9bc49c95a4b108 on tablet 1280b2f88bbc470bbc9bc49c95a4b108 with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:29.927269 30836 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:29.927086 32301 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:29.944003 30563 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:29.957405 30967 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:29.968336 30703 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 119. Preceding OpId from leader: term: 1 index: 121. (index mismatch)
I20240501 08:11:29.978652 31355 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 119. Preceding OpId from leader: term: 1 index: 121. (index mismatch)
W20240501 08:11:30.013626 31295 consensus_peers.cc:485] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
I20240501 08:11:30.014993 578 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 120, Last known committed idx: 119, Time since last communication: 0.000s
W20240501 08:11:30.025782 31294 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 143309us
I20240501 08:11:30.045440 32301 raft_consensus.cc:1203] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.109->[1.110-1.136] Dedup: 1.122->[1.123-1.136]
I20240501 08:11:30.046228 420 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=124 overwritten=0 applied=124 ignored=0} inserts{seen=809 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:30.047657 420 tablet_bootstrap.cc:492] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:30.049625 420 ts_tablet_manager.cc:1367] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 2.646s user 0.186s sys 0.014s
I20240501 08:11:30.058207 578 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 120, Last known committed idx: 119, Time since last communication: 0.000s
I20240501 08:11:30.058816 420 raft_consensus.cc:348] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:30.059072 420 raft_consensus.cc:729] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:30.066025 420 consensus_queue.cc:260] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 124, Last appended: 1.124, Last appended by leader: 124, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 107 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:30.068593 31357 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.071235 420 ts_tablet_manager.cc:1398] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.021s user 0.002s sys 0.000s
I20240501 08:11:30.073246 31380 tablet_copy_service.cc:342] P 6f37f9ed957342dba1271066c1bbeade: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-120d5a7fb6704b4eb07c02ab8dabf6cd received from {username='slave'} at 127.28.1.135:56725
I20240501 08:11:30.073423 31380 tablet_copy_service.cc:433] P 6f37f9ed957342dba1271066c1bbeade: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-120d5a7fb6704b4eb07c02ab8dabf6cd on tablet 120d5a7fb6704b4eb07c02ab8dabf6cd with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:30.128418 32301 raft_consensus.cc:1203] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.100->[1.101-1.155] Dedup: 1.122->[1.123-1.155]
I20240501 08:11:30.219988 367 raft_consensus.cc:1050] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:30.221207 367 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 156, Committed index: 156, Last appended: 1.156, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.224480 30704 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240501 08:11:30.225062 320 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 156, Time since last communication: 0.000s
I20240501 08:11:30.225620 32301 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240501 08:11:30.226696 320 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 156, Time since last communication: 0.000s
I20240501 08:11:30.233217 31229 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240501 08:11:30.234143 31095 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240501 08:11:30.235164 32712 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.000s
I20240501 08:11:30.247581 32712 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 156, Time since last communication: 0.000s
I20240501 08:11:30.283910 464 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.289798 30840 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 154. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240501 08:11:30.290503 30709 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.292384 30561 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:30.295776 32301 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.296813 367 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 153, Time since last communication: 0.000s
I20240501 08:11:30.302397 32300 raft_consensus.cc:1203] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.114->[1.115-1.134] Dedup: 1.121->[1.122-1.134]
I20240501 08:11:30.317409 31229 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.366041 30837 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.366643 30703 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.120: config changed from index 90 to 120, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 120 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.386708 30968 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 158, Committed index: 158, Last appended: 1.159, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.389292 31095 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 101 to 157, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.390226 32301 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:30.390882 30837 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:30.390812 532 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 158, Time since last communication: 0.000s
I20240501 08:11:30.391474 532 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 158, Time since last communication: 0.000s
I20240501 08:11:30.393883 532 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.160: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.398510 32301 raft_consensus.cc:1203] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.106->[1.107-1.145] Dedup: 1.124->[1.125-1.145]
I20240501 08:11:30.400887 30837 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.402051 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8302dd0a0d2e4a20bdc00e27da939e8a with cas_config_opid_index 157: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:30.402525 30561 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:30.403162 32299 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.407567 30687 tablet_service.cc:1514] Processing DeleteTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a with delete_type TABLET_DATA_TOMBSTONED (TS 3170be3b2bbb4b3bbd8a37ac5f91de89 not found in new config with opid_index 160) from {username='slave'} at 127.0.0.1:43428
I20240501 08:11:30.423377 31096 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 158. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:30.424326 602 tablet_replica.cc:330] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: stopping tablet replica
I20240501 08:11:30.429232 602 raft_consensus.cc:2229] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:30.432806 602 pending_rounds.cc:62] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Trying to abort 1 pending ops.
I20240501 08:11:30.444043 602 pending_rounds.cc:68] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Aborting op as it isn't in flight: id { term: 1 index: 159 } timestamp: 7022801266190155776 op_type: WRITE_OP write_request { tablet_id: "8302dd0a0d2e4a20bdc00e27da939e8a" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\0006D\353-\327\265\035\363\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""cd84a3687e080f8d" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801266111221760 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n 3881d7493a564e21be970e6b9216ac77\020\001\030\001 \001(\001" signature: "<redacted>""\013mka\023\010\032t\014\347a\036t\342\2047\254\370\355\313\322\302?\332\267\337\357\210\211D\331]IB\240\306P\301\024\353\363\001C\315[\276{Qm\004\275\363A\307\221O\263\265d\342\257Q\203m" signing_key_seq_num: 0 } } request_id { client_id: "b0bc06be9d8a4dfbb23da77c3ebfa27d" seq_no: 502 first_incomplete_seq_no: 502 attempt_no: 0 }
I20240501 08:11:30.445684 602 raft_consensus.cc:2258] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:30.448959 602 ts_tablet_manager.cc:1875] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:30.455493 532 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 158, Time since last communication: 0.000s
I20240501 08:11:30.456528 31228 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:30.458487 31095 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.475086 464 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 158, Time since last communication: 0.002s
I20240501 08:11:30.488880 602 ts_tablet_manager.cc:1888] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.159
I20240501 08:11:30.493183 602 log.cc:1196] T 8302dd0a0d2e4a20bdc00e27da939e8a P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:11:30.564958 31228 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index 157 to 160, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.569630 32548 raft_consensus.cc:1050] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:30.574553 32548 consensus_queue.cc:237] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 145, Committed index: 145, Last appended: 1.146, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.578718 32301 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 146. Preceding OpId from leader: term: 1 index: 147. (index mismatch)
I20240501 08:11:30.579437 579 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 147, Last known committed idx: 145, Time since last communication: 0.000s
I20240501 08:11:30.588634 30840 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 145. Preceding OpId from leader: term: 1 index: 147. (index mismatch)
I20240501 08:11:30.588780 31099 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 146. Preceding OpId from leader: term: 1 index: 147. (index mismatch)
I20240501 08:11:30.589280 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 147, Last known committed idx: 145, Time since last communication: 0.000s
I20240501 08:11:30.589943 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 147, Last known committed idx: 145, Time since last communication: 0.000s
I20240501 08:11:30.590893 495 raft_consensus.cc:1050] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:30.591377 30968 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 146. Preceding OpId from leader: term: 1 index: 147. (index mismatch)
I20240501 08:11:30.591884 32548 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 147, Last known committed idx: 145, Time since last communication: 0.000s
I20240501 08:11:30.592243 495 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 134, Committed index: 134, Last appended: 1.135, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.596902 31355 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 135. Preceding OpId from leader: term: 1 index: 136. (index mismatch)
I20240501 08:11:30.597554 495 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 136, Last known committed idx: 134, Time since last communication: 0.000s
I20240501 08:11:30.599937 32300 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 135. Preceding OpId from leader: term: 1 index: 136. (index mismatch)
I20240501 08:11:30.600365 31099 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 135. Preceding OpId from leader: term: 1 index: 136. (index mismatch)
I20240501 08:11:30.600205 577 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 LEADER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.602834 32494 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 136, Last known committed idx: 134, Time since last communication: 0.000s
I20240501 08:11:30.604694 30840 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.608355 31096 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.613304 30561 catalog_manager.cc:5484] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade reported cstate change: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6f37f9ed957342dba1271066c1bbeade" committed_config { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:30.621999 31230 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 146. Preceding OpId from leader: term: 1 index: 147. (index mismatch)
I20240501 08:11:30.626112 30835 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 134. Preceding OpId from leader: term: 1 index: 136. (index mismatch)
I20240501 08:11:30.604878 32301 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.630630 588 raft_consensus.cc:1050] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:30.631757 588 consensus_queue.cc:237] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 138, Committed index: 138, Last appended: 1.138, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.634296 30834 raft_consensus.cc:1203] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.145->[1.146-1.146] Dedup: 1.146->[]
I20240501 08:11:30.635816 32301 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240501 08:11:30.637241 445 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Bootstrap replayed 1/1 log segments. Stats: ops{read=142 overwritten=0 applied=141 ignored=0} inserts{seen=772 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240501 08:11:30.638038 445 tablet_bootstrap.cc:492] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Bootstrap complete.
I20240501 08:11:30.638299 600 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:30.641604 445 ts_tablet_manager.cc:1367] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Time spent bootstrapping tablet: real 3.651s user 0.194s sys 0.035s
I20240501 08:11:30.644001 30969 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240501 08:11:30.644693 600 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:30.649055 31358 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240501 08:11:30.649729 603 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:30.651333 32494 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 136, Last known committed idx: 134, Time since last communication: 0.000s
I20240501 08:11:30.654058 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 147, Last known committed idx: 145, Time since last communication: 0.000s
I20240501 08:11:30.654808 32422 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 136, Last known committed idx: 134, Time since last communication: 0.000s
I20240501 08:11:30.657879 445 raft_consensus.cc:348] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } }
I20240501 08:11:30.658710 445 raft_consensus.cc:729] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Initialized, Role: LEARNER
I20240501 08:11:30.663158 445 consensus_queue.cc:260] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 141, Last appended: 1.142, Last appended by leader: 142, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 133 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: true } }
I20240501 08:11:30.663038 573 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.668413 30835 raft_consensus.cc:1203] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.134->[1.135-1.136] Dedup: 1.135->[1.136-1.136]
I20240501 08:11:30.672873 32299 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.681910 30561 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.682055 31354 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.678548 30835 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.684998 445 ts_tablet_manager.cc:1398] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138: Time spent starting tablet: real 0.031s user 0.002s sys 0.000s
I20240501 08:11:30.685895 30836 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240501 08:11:30.688732 603 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:30.705873 537 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 LEADER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.713433 32299 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.723814 30840 raft_consensus.cc:1203] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.132->[1.133-1.188] Dedup: 1.142->[1.143-1.188]
I20240501 08:11:30.724042 30567 catalog_manager.cc:5484] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.727246 30835 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.739866 30709 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240501 08:11:30.742714 31228 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.749423 30990 tablet_copy_service.cc:342] P 57210dc0a07a4851a43be23a279eb836: Request end of tablet copy session 1a388864321846929919a37b570e8138-9c025c9ec2c74a54ab24c40ed11639f6 received from {username='slave'} at 127.28.1.130:49241
I20240501 08:11:30.749572 30990 tablet_copy_service.cc:433] P 57210dc0a07a4851a43be23a279eb836: ending tablet copy session 1a388864321846929919a37b570e8138-9c025c9ec2c74a54ab24c40ed11639f6 on tablet 9c025c9ec2c74a54ab24c40ed11639f6 with peer 1a388864321846929919a37b570e8138
I20240501 08:11:30.753615 31355 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.759439 31354 consensus_queue.cc:237] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 148, Committed index: 148, Last appended: 1.149, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.760193 30968 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.147: config changed from index 107 to 147, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 147 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.762663 31096 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:30.775691 31033 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 126561us
I20240501 08:11:30.777037 600 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:30.779331 30709 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.781149 32300 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 149. Preceding OpId from leader: term: 1 index: 150. (index mismatch)
I20240501 08:11:30.781759 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 150, Last known committed idx: 148, Time since last communication: 0.000s
I20240501 08:11:30.785367 30834 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 149. Preceding OpId from leader: term: 1 index: 150. (index mismatch)
I20240501 08:11:30.786104 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 150, Last known committed idx: 148, Time since last communication: 0.000s
I20240501 08:11:30.790154 577 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [term 1 LEADER]: Committing config change with OpId 1.150: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.794154 30552 catalog_manager.cc:4830] TS 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): tablet 8302dd0a0d2e4a20bdc00e27da939e8a (table rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]) successfully deleted
I20240501 08:11:30.796129 30554 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd with cas_config_opid_index 147: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:30.798804 30834 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.150: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.800658 31098 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 148. Preceding OpId from leader: term: 1 index: 150. (index mismatch)
W20240501 08:11:30.800688 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 97603.5us
I20240501 08:11:30.801234 579 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 150, Last known committed idx: 148, Time since last communication: 0.000s
I20240501 08:11:30.804193 31098 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.150: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.805757 30706 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 135. Preceding OpId from leader: term: 1 index: 136. (index mismatch)
I20240501 08:11:30.817728 30969 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index 110 to 139, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.819056 32300 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.150: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.828549 30561 catalog_manager.cc:5484] T 120d5a7fb6704b4eb07c02ab8dabf6cd P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "6f37f9ed957342dba1271066c1bbeade" committed_config { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.833535 31209 tablet_service.cc:1514] Processing DeleteTablet for tablet 120d5a7fb6704b4eb07c02ab8dabf6cd with delete_type TABLET_DATA_TOMBSTONED (TS 9204a82012ac492dbc0da9c84c51dabf not found in new config with opid_index 150) from {username='slave'} at 127.0.0.1:57760
I20240501 08:11:30.840889 31093 consensus_queue.cc:237] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 140, Committed index: 140, Last appended: 1.140, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:30.844208 30709 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 140. Preceding OpId from leader: term: 1 index: 141. (index mismatch)
I20240501 08:11:30.844316 32299 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 140. Preceding OpId from leader: term: 1 index: 141. (index mismatch)
I20240501 08:11:30.845109 30969 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 140. Preceding OpId from leader: term: 1 index: 141. (index mismatch)
I20240501 08:11:30.848470 601 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 141, Last known committed idx: 140, Time since last communication: 0.000s
I20240501 08:11:30.853266 603 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 141, Last known committed idx: 140, Time since last communication: 0.000s
I20240501 08:11:30.857053 603 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 141, Last known committed idx: 140, Time since last communication: 0.000s
I20240501 08:11:30.868973 30837 raft_consensus.cc:1261] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 140. Preceding OpId from leader: term: 1 index: 141. (index mismatch)
I20240501 08:11:30.869699 601 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 LEADER]: Committing config change with OpId 1.141: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.881183 588 consensus_queue.cc:1035] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 141, Last known committed idx: 139, Time since last communication: 0.000s
I20240501 08:11:30.884084 32300 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.141: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.891259 30565 catalog_manager.cc:5484] T 1280b2f88bbc470bbc9bc49c95a4b108 P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New cstate: current_term: 1 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.893769 30835 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.141: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.895835 31338 tablet_service.cc:1514] Processing DeleteTablet for tablet 1280b2f88bbc470bbc9bc49c95a4b108 with delete_type TABLET_DATA_TOMBSTONED (TS 6f37f9ed957342dba1271066c1bbeade not found in new config with opid_index 141) from {username='slave'} at 127.0.0.1:43272
I20240501 08:11:30.899089 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1280b2f88bbc470bbc9bc49c95a4b108 with cas_config_opid_index 139: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:30.899624 30969 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.141: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.907459 30967 raft_consensus.cc:1261] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 6f37f9ed957342dba1271066c1bbeade: Log matching property violated. Preceding OpId in replica: term: 1 index: 149. Preceding OpId from leader: term: 1 index: 150. (index mismatch)
W20240501 08:11:30.909143 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 155770us
I20240501 08:11:30.911346 577 consensus_queue.cc:1035] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 150, Last known committed idx: 148, Time since last communication: 0.000s
W20240501 08:11:30.912405 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 160231us
I20240501 08:11:30.917140 573 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 136, Last known committed idx: 134, Time since last communication: 0.000s
I20240501 08:11:30.918859 30970 raft_consensus.cc:2932] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.150: config changed from index 147 to 150, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 150 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:30.997485 628 tablet_replica.cc:330] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: stopping tablet replica
I20240501 08:11:30.998402 628 raft_consensus.cc:2229] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:31.001206 628 raft_consensus.cc:2258] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:31.067849 627 tablet_replica.cc:330] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: stopping tablet replica
I20240501 08:11:31.068382 627 raft_consensus.cc:2229] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:31.068740 627 pending_rounds.cc:62] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Trying to abort 1 pending ops.
I20240501 08:11:31.068850 627 pending_rounds.cc:68] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Aborting op as it isn't in flight: id { term: 1 index: 148 } timestamp: 7022801267110920192 op_type: WRITE_OP write_request { tablet_id: "120d5a7fb6704b4eb07c02ab8dabf6cd" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\000\312\211D-\031\177\242\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""d2c142719be26652" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801266947698688 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n e93d8483bead454d8e47e99f73d1dabb\020\001\030\001 \001(\001" signature: "<redacted>""\272O\355\373\2120\201?\215\020O\361\323\366\223 f|\327o\347\004\254o\005%\033\304\262v\207\215\312Y\301gu\262\306\255O,a\035\247\306\033=\200.\257\262\277\006v,\233\224\003\370\247\330w\212" signing_key_seq_num: 0 } } request_id { client_id: "c52c7e7b69284539914688f45a543b0c" seq_no: 463 first_incomplete_seq_no: 463 attempt_no: 0 }
I20240501 08:11:31.069561 627 raft_consensus.cc:2258] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:31.072207 627 ts_tablet_manager.cc:1875] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:31.075831 30703 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.136: config changed from index 115 to 136, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 136 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.076586 30706 raft_consensus.cc:2932] T 1280b2f88bbc470bbc9bc49c95a4b108 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.141: config changed from index 139 to 141, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 141 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.118033 627 ts_tablet_manager.cc:1888] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.148
I20240501 08:11:31.133088 627 log.cc:1196] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 9204a82012ac492dbc0da9c84c51dabf: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:11:31.138209 30553 catalog_manager.cc:4830] TS 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): tablet 120d5a7fb6704b4eb07c02ab8dabf6cd (table rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]) successfully deleted
I20240501 08:11:31.163599 31225 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 138, Committed index: 138, Last appended: 1.139, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:31.170704 32301 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 140. (index mismatch)
I20240501 08:11:31.173097 495 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 140, Last known committed idx: 137, Time since last communication: 0.000s
I20240501 08:11:31.178889 30836 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 139. Preceding OpId from leader: term: 1 index: 140. (index mismatch)
I20240501 08:11:31.180264 495 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 140, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:31.174204 446 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap replayed 1/1 log segments. Stats: ops{read=138 overwritten=0 applied=137 ignored=0} inserts{seen=768 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240501 08:11:31.181118 446 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Bootstrap complete.
I20240501 08:11:31.181594 446 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent bootstrapping tablet: real 4.037s user 0.203s sys 0.037s
I20240501 08:11:31.182469 446 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } }
I20240501 08:11:31.185940 32494 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.140: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.186386 446 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3170be3b2bbb4b3bbd8a37ac5f91de89, State: Initialized, Role: LEARNER
I20240501 08:11:31.186995 446 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 137, Last appended: 1.138, Last appended by leader: 138, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: NON_VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: true } }
I20240501 08:11:31.187718 446 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:11:31.190892 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bc20191679084ecfbd360e23d68abd62 with cas_config_opid_index 136: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:31.192631 30837 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.140: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.198485 32299 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.140: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.203678 30565 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:31.209620 31079 tablet_service.cc:1514] Processing DeleteTablet for tablet bc20191679084ecfbd360e23d68abd62 with delete_type TABLET_DATA_TOMBSTONED (TS 9fc0640dbc0d4208bfb536949d1bb0c4 not found in new config with opid_index 140) from {username='slave'} at 127.0.0.1:43748
I20240501 08:11:31.210026 31230 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 125, Committed index: 125, Last appended: 1.125, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:31.220407 32301 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 125. Preceding OpId from leader: term: 1 index: 126. (index mismatch)
I20240501 08:11:31.221694 494 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 126, Last known committed idx: 125, Time since last communication: 0.000s
I20240501 08:11:31.227440 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 3170be3b2bbb4b3bbd8a37ac5f91de89-5730faa05092410da14e00e24ad74c86 received from {username='slave'} at 127.28.1.129:57131
I20240501 08:11:31.227619 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 3170be3b2bbb4b3bbd8a37ac5f91de89-5730faa05092410da14e00e24ad74c86 on tablet 5730faa05092410da14e00e24ad74c86 with peer 3170be3b2bbb4b3bbd8a37ac5f91de89
I20240501 08:11:31.230885 30706 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 139. Preceding OpId from leader: term: 1 index: 140. (index mismatch)
I20240501 08:11:31.234362 30970 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 125. Preceding OpId from leader: term: 1 index: 126. (index mismatch)
I20240501 08:11:31.234449 642 tablet_replica.cc:330] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: stopping tablet replica
I20240501 08:11:31.234972 642 raft_consensus.cc:2229] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:31.235991 642 pending_rounds.cc:62] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Trying to abort 1 pending ops.
I20240501 08:11:31.236155 642 pending_rounds.cc:68] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Aborting op as it isn't in flight: id { term: 1 index: 139 } timestamp: 7022801269293297664 op_type: WRITE_OP write_request { tablet_id: "bc20191679084ecfbd360e23d68abd62" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\037\220\256$\360\351\271\203\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""18118a5c4f2554b8" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801269100707840 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n d57de712d6664d73a00cf72cfbe56003\020\001\030\001 \001(\001" signature: "<redacted>""\234\007\230P\375\217\001\034E\323\021\246E0\r\227\\R5L\262{\225\026J\247\303\204\276\027\372\335\371*\2343\311\362\346\270k\216\201\250\257\362\204E\336\372me@Ai\345a\322\266\n\266G3\202" signing_key_seq_num: 0 } } request_id { client_id: "603753885a3a4e4fbc24bad429efb7ba" seq_no: 387 first_incomplete_seq_no: 387 attempt_no: 0 }
I20240501 08:11:31.236850 30837 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 125. Preceding OpId from leader: term: 1 index: 126. (index mismatch)
I20240501 08:11:31.237011 495 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 126, Last known committed idx: 125, Time since last communication: 0.000s
I20240501 08:11:31.238507 642 raft_consensus.cc:2258] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:31.241367 642 ts_tablet_manager.cc:1875] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:31.238196 629 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 126, Last known committed idx: 125, Time since last communication: 0.000s
I20240501 08:11:31.256268 31355 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 139. Preceding OpId from leader: term: 1 index: 140. (index mismatch)
I20240501 08:11:31.256806 578 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.126: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.261792 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with cas_config_opid_index 120: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:31.263231 494 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 140, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:31.264762 30970 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.126: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.265753 628 ts_tablet_manager.cc:1875] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:31.279006 30561 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:31.281842 32300 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.126: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.285065 31338 tablet_service.cc:1514] Processing DeleteTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with delete_type TABLET_DATA_TOMBSTONED (TS 6f37f9ed957342dba1271066c1bbeade not found in new config with opid_index 126) from {username='slave'} at 127.0.0.1:43272
I20240501 08:11:31.293642 30709 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.140: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.294724 642 ts_tablet_manager.cc:1888] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.139
I20240501 08:11:31.305296 642 log.cc:1196] T bc20191679084ecfbd360e23d68abd62 P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/bc20191679084ecfbd360e23d68abd62
I20240501 08:11:31.306444 30553 catalog_manager.cc:4830] TS 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): tablet bc20191679084ecfbd360e23d68abd62 (table rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]) successfully deleted
I20240501 08:11:31.307538 495 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 140, Last known committed idx: 138, Time since last communication: 0.000s
I20240501 08:11:31.315160 30709 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 125. Preceding OpId from leader: term: 1 index: 126. (index mismatch)
I20240501 08:11:31.316759 629 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 126, Last known committed idx: 125, Time since last communication: 0.000s
I20240501 08:11:31.357846 32712 raft_consensus.cc:1050] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836: attempting to promote NON_VOTER 1a388864321846929919a37b570e8138 to VOTER
I20240501 08:11:31.358740 32712 consensus_queue.cc:237] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 195, Committed index: 195, Last appended: 1.195, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } }
I20240501 08:11:31.361692 30703 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 195. Preceding OpId from leader: term: 1 index: 196. (index mismatch)
I20240501 08:11:31.362383 464 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 196, Last known committed idx: 195, Time since last communication: 0.000s
I20240501 08:11:31.364480 30836 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 LEARNER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 193. Preceding OpId from leader: term: 1 index: 196. (index mismatch)
I20240501 08:11:31.367141 31094 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 195. Preceding OpId from leader: term: 1 index: 196. (index mismatch)
I20240501 08:11:31.367686 32712 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 196, Last known committed idx: 195, Time since last communication: 0.000s
I20240501 08:11:31.370251 32712 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 196, Last known committed idx: 193, Time since last communication: 0.000s
I20240501 08:11:31.350700 30835 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.126: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.376919 30703 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.126: config changed from index 120 to 126, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 126 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.380057 31358 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 195. Preceding OpId from leader: term: 1 index: 196. (index mismatch)
I20240501 08:11:31.380625 366 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 196, Last known committed idx: 195, Time since last communication: 0.000s
I20240501 08:11:31.383198 30836 raft_consensus.cc:1203] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.193->[1.194-1.196] Dedup: 1.195->[1.196-1.196]
I20240501 08:11:31.384856 32712 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.385780 30840 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.386047 31356 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.391772 30709 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.394493 31094 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.400681 30561 catalog_manager.cc:5484] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.438678 31356 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.140: config changed from index 136 to 140, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 140 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:31.473605 31227 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 194. Preceding OpId from leader: term: 1 index: 196. (index mismatch)
I20240501 08:11:31.495556 30709 raft_consensus.cc:1203] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.128->[1.129-1.176] Dedup: 1.138->[1.139-1.176]
I20240501 08:11:31.584203 366 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 196, Last known committed idx: 193, Time since last communication: 0.000s
I20240501 08:11:31.625025 31226 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.196: config changed from index 133 to 196, 1a388864321846929919a37b570e8138 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 196 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
W20240501 08:11:31.701045 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 77595.5us
I20240501 08:11:31.747612 628 ts_tablet_manager.cc:1888] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.140
I20240501 08:11:31.747803 628 log.cc:1196] T 1280b2f88bbc470bbc9bc49c95a4b108 P 6f37f9ed957342dba1271066c1bbeade: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/wals/1280b2f88bbc470bbc9bc49c95a4b108
W20240501 08:11:31.766777 366 consensus_queue.cc:1159] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" responder_term: 1 status { last_received { term: 1 index: 198 } last_committed_idx: 197 last_received_current_leader { term: 1 index: 198 } }
I20240501 08:11:31.767324 30967 consensus_queue.cc:237] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 197, Committed index: 197, Last appended: 1.198, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } }
I20240501 08:11:31.771190 30836 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 198. Preceding OpId from leader: term: 1 index: 199. (index mismatch)
I20240501 08:11:31.772423 596 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 199, Last known committed idx: 197, Time since last communication: 0.000s
I20240501 08:11:31.774482 31227 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 198. Preceding OpId from leader: term: 1 index: 199. (index mismatch)
I20240501 08:11:31.776732 596 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 199, Last known committed idx: 197, Time since last communication: 0.000s
I20240501 08:11:31.800246 30704 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 197. Preceding OpId from leader: term: 1 index: 199. (index mismatch)
I20240501 08:11:31.805874 596 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 199, Last known committed idx: 197, Time since last communication: 0.000s
I20240501 08:11:31.819476 31358 raft_consensus.cc:1261] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 198. Preceding OpId from leader: term: 1 index: 199. (index mismatch)
I20240501 08:11:31.824652 30703 raft_consensus.cc:1203] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.197->[1.198-1.199] Dedup: 1.198->[1.199-1.199]
I20240501 08:11:31.834766 366 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.199: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.839162 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9c025c9ec2c74a54ab24c40ed11639f6 with cas_config_opid_index 196: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:31.841049 30835 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.199: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.845817 30567 catalog_manager.cc:5484] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:31.846917 30703 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.199: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.851385 31079 tablet_service.cc:1514] Processing DeleteTablet for tablet 9c025c9ec2c74a54ab24c40ed11639f6 with delete_type TABLET_DATA_TOMBSTONED (TS 9fc0640dbc0d4208bfb536949d1bb0c4 not found in new config with opid_index 199) from {username='slave'} at 127.0.0.1:43748
I20240501 08:11:31.889919 366 consensus_queue.cc:1035] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 199, Last known committed idx: 197, Time since last communication: 0.000s
I20240501 08:11:31.896634 32494 raft_consensus.cc:1050] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 to VOTER
I20240501 08:11:31.897627 32494 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 183, Committed index: 183, Last appended: 1.183, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } }
I20240501 08:11:31.901332 30709 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 183. Preceding OpId from leader: term: 1 index: 184. (index mismatch)
I20240501 08:11:31.901870 629 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 184, Last known committed idx: 183, Time since last communication: 0.000s
I20240501 08:11:31.909533 628 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:43272 (request call id 14) took 1014 ms. Trace:
I20240501 08:11:31.919601 31094 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 183. Preceding OpId from leader: term: 1 index: 184. (index mismatch)
I20240501 08:11:31.921399 32494 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 184, Last known committed idx: 183, Time since last communication: 0.000s
I20240501 08:11:31.934804 30967 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 183. Preceding OpId from leader: term: 1 index: 184. (index mismatch)
I20240501 08:11:31.938160 32494 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 184, Last known committed idx: 183, Time since last communication: 0.000s
I20240501 08:11:31.911924 628 rpcz_store.cc:276] 0501 08:11:30.895540 (+ 0us) service_pool.cc:168] Inserting onto call queue
0501 08:11:30.895597 (+ 57us) service_pool.cc:225] Handling call
0501 08:11:30.997150 (+101553us) ts_tablet_manager.cc:1120] Deleting tablet 1280b2f88bbc470bbc9bc49c95a4b108
0501 08:11:30.997165 (+ 15us) ts_tablet_manager.cc:1040] Acquired tablet manager lock
0501 08:11:31.624545 (+627380us) tablet_metadata.cc:641] Metadata flushed
0501 08:11:31.747441 (+122896us) tablet_metadata.cc:641] Metadata flushed
0501 08:11:31.909465 (+162024us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":100664,"thread_start_us":101513,"threads_started":1}
I20240501 08:11:31.948145 628 tablet_replica.cc:330] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: stopping tablet replica
I20240501 08:11:31.948148 30554 catalog_manager.cc:4830] TS 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): tablet 1280b2f88bbc470bbc9bc49c95a4b108 (table rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]) successfully deleted
I20240501 08:11:31.948375 628 raft_consensus.cc:2229] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:31.952132 30837 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 182. Preceding OpId from leader: term: 1 index: 184. (index mismatch)
I20240501 08:11:31.953629 629 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 184, Last known committed idx: 182, Time since last communication: 0.000s
I20240501 08:11:31.958432 628 raft_consensus.cc:2258] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:31.962000 628 ts_tablet_manager.cc:1875] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:31.964329 31227 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.199: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:31.973791 31353 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 182. Preceding OpId from leader: term: 1 index: 184. (index mismatch)
I20240501 08:11:32.004238 629 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 184, Last known committed idx: 181, Time since last communication: 0.000s
I20240501 08:11:32.006296 629 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.008018 31352 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.011921 31100 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.014742 30561 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade reported cstate change: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.017306 30837 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.025900 628 ts_tablet_manager.cc:1888] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.125
I20240501 08:11:32.026132 628 log.cc:1196] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 6f37f9ed957342dba1271066c1bbeade: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/wals/3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:11:32.026752 30707 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.045164 30969 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.184: config changed from index 129 to 184, 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) changed from NON_VOTER to VOTER. New config: { opid_index: 184 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.065930 30554 catalog_manager.cc:4830] TS 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (table rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]) successfully deleted
I20240501 08:11:32.080339 30709 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 168, Committed index: 168, Last appended: 1.169, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:32.131759 31230 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 185, Committed index: 185, Last appended: 1.185, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } }
I20240501 08:11:32.159417 31099 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240501 08:11:32.159924 32494 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.002s
I20240501 08:11:32.163343 31354 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240501 08:11:32.163897 578 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.000s
I20240501 08:11:32.173573 32494 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.186: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.174176 31099 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.177659 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5730faa05092410da14e00e24ad74c86 with cas_config_opid_index 184: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:32.183909 31354 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.184928 30565 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.186105 667 tablet_replica.cc:330] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: stopping tablet replica
W20240501 08:11:32.186604 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 65665.2us
I20240501 08:11:32.188186 30703 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240501 08:11:32.196277 32422 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 184, Time since last communication: 0.000s
I20240501 08:11:32.199328 30706 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.207312 30948 tablet_service.cc:1514] Processing DeleteTablet for tablet 5730faa05092410da14e00e24ad74c86 with delete_type TABLET_DATA_TOMBSTONED (TS 57210dc0a07a4851a43be23a279eb836 not found in new config with opid_index 186) from {username='slave'} at 127.0.0.1:44190
I20240501 08:11:32.240990 31230 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 168. Preceding OpId from leader: term: 1 index: 170. (index mismatch)
I20240501 08:11:32.241107 30834 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 169. Preceding OpId from leader: term: 1 index: 170. (index mismatch)
W20240501 08:11:32.242300 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 100638us
I20240501 08:11:32.241619 564 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 170, Last known committed idx: 167, Time since last communication: 0.001s
I20240501 08:11:32.242709 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 170, Last known committed idx: 168, Time since last communication: 0.000s
I20240501 08:11:32.215688 31359 raft_consensus.cc:2932] T 9c025c9ec2c74a54ab24c40ed11639f6 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.199: config changed from index 196 to 199, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 199 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { promote: false } } }
I20240501 08:11:32.188758 31094 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 169. Preceding OpId from leader: term: 1 index: 170. (index mismatch)
I20240501 08:11:32.250207 30836 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240501 08:11:32.261806 667 raft_consensus.cc:2229] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:32.262310 667 pending_rounds.cc:62] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Trying to abort 1 pending ops.
I20240501 08:11:32.262394 667 pending_rounds.cc:68] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Aborting op as it isn't in flight: id { term: 1 index: 198 } timestamp: 7022801271856467968 op_type: WRITE_OP write_request { tablet_id: "9c025c9ec2c74a54ab24c40ed11639f6" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\333\212kC^\236\243\241\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""c1f511aa00b561ce" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801271525359616 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n 3881d7493a564e21be970e6b9216ac77\020\001\030\001 \001(\001" signature: "<redacted>""\013mka\023\010\032t\014\347a\036t\342\2047\254\370\355\313\322\302?\332\267\337\357\210\211D\331]IB\240\306P\301\024\353\363\001C\315[\276{Qm\004\275\363A\307\221O\263\265d\342\257Q\203m" signing_key_seq_num: 0 } } request_id { client_id: "b0bc06be9d8a4dfbb23da77c3ebfa27d" seq_no: 538 first_incomplete_seq_no: 538 attempt_no: 0 }
I20240501 08:11:32.262986 667 raft_consensus.cc:2258] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:32.263444 675 tablet_replica.cc:330] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: stopping tablet replica
I20240501 08:11:32.264041 629 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.001s
I20240501 08:11:32.265528 667 ts_tablet_manager.cc:1875] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:32.267093 675 raft_consensus.cc:2229] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:32.282191 675 raft_consensus.cc:2258] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:32.286526 31354 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 168. Preceding OpId from leader: term: 1 index: 170. (index mismatch)
I20240501 08:11:32.287259 564 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 170, Last known committed idx: 168, Time since last communication: 0.000s
I20240501 08:11:32.288020 30837 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 184 to 186, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false } } }
I20240501 08:11:32.297840 612 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.170: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:32.299120 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 170, Last known committed idx: 167, Time since last communication: 0.000s
I20240501 08:11:32.303357 31354 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.170: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:32.304611 31294 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 47211.6us
I20240501 08:11:32.321770 30561 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:32.326406 31095 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.170: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:32.355051 667 ts_tablet_manager.cc:1888] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.198
I20240501 08:11:32.356276 667 log.cc:1196] T 9c025c9ec2c74a54ab24c40ed11639f6 P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:11:32.358194 675 ts_tablet_manager.cc:1875] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:32.372298 30553 catalog_manager.cc:4830] TS 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): tablet 9c025c9ec2c74a54ab24c40ed11639f6 (table rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]) successfully deleted
W20240501 08:11:32.377418 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 59959.7us
I20240501 08:11:32.376116 31227 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.170: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:32.388278 30644 consensus_peers.cc:485] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b3a5e3d4cc604fcd90a0efeea3946acd. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:32.397238 30835 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.170: config changed from index -1 to 170, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:32.475370 675 ts_tablet_manager.cc:1888] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.185
I20240501 08:11:32.476210 675 log.cc:1196] T 5730faa05092410da14e00e24ad74c86 P 57210dc0a07a4851a43be23a279eb836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/5730faa05092410da14e00e24ad74c86
I20240501 08:11:32.477710 30552 catalog_manager.cc:4830] TS 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): tablet 5730faa05092410da14e00e24ad74c86 (table rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]) successfully deleted
W20240501 08:11:32.620759 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 98247.1us
I20240501 08:11:32.720280 689 ts_tablet_manager.cc:902] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637)
W20240501 08:11:32.766970 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 54510.7us
I20240501 08:11:32.866669 689 tablet_copy_client.cc:308] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:39637
W20240501 08:11:32.889634 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 44832.5us
I20240501 08:11:33.007220 30729 tablet_copy_service.cc:140] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Received BeginTabletCopySession request for tablet b3a5e3d4cc604fcd90a0efeea3946acd from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:36743)
I20240501 08:11:33.007472 30729 tablet_copy_service.cc:161] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Beginning new tablet copy session on tablet b3a5e3d4cc604fcd90a0efeea3946acd from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:36743: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:11:33.013001 30729 tablet_copy_source_session.cc:205] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:33.014948 689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:33.019934 689 tablet_copy_client.cc:783] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:33.020174 689 tablet_copy_client.cc:647] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:33.024258 689 tablet_copy_client.cc:523] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:33.029753 689 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
W20240501 08:11:33.040254 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 64926.5us
W20240501 08:11:33.373528 30903 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 45112.5us
W20240501 08:11:33.421993 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 93721us
W20240501 08:11:34.148100 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 86982.2us
I20240501 08:11:34.246445 689 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=179 overwritten=0 applied=179 ignored=0} inserts{seen=851 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:34.247570 689 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:34.248900 689 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 1.219s user 0.215s sys 0.052s
I20240501 08:11:34.251323 689 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:34.252624 689 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:34.253083 689 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 179, Last appended: 1.179, Last appended by leader: 179, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 170 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:34.261446 689 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.012s user 0.002s sys 0.003s
I20240501 08:11:34.271446 30729 tablet_copy_service.cc:342] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-b3a5e3d4cc604fcd90a0efeea3946acd received from {username='slave'} at 127.28.1.135:36743
I20240501 08:11:34.271680 30729 tablet_copy_service.cc:433] P 3170be3b2bbb4b3bbd8a37ac5f91de89: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-b3a5e3d4cc604fcd90a0efeea3946acd on tablet b3a5e3d4cc604fcd90a0efeea3946acd with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:34.345424 32299 raft_consensus.cc:1203] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.169->[1.170-1.188] Dedup: 1.179->[1.180-1.188]
I20240501 08:11:34.469789 714 raft_consensus.cc:1050] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:34.470814 714 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 190, Committed index: 190, Last appended: 1.191, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:34.479346 32299 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 189. Preceding OpId from leader: term: 1 index: 192. (index mismatch)
I20240501 08:11:34.479916 734 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 192, Last known committed idx: 188, Time since last communication: 0.000s
I20240501 08:11:34.484007 31095 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 191. Preceding OpId from leader: term: 1 index: 192. (index mismatch)
I20240501 08:11:34.484722 734 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 192, Last known committed idx: 190, Time since last communication: 0.000s
I20240501 08:11:34.502651 31230 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 192. (index mismatch)
I20240501 08:11:34.514761 734 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 192, Last known committed idx: 189, Time since last communication: 0.000s
I20240501 08:11:34.531466 30837 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 191. Preceding OpId from leader: term: 1 index: 192. (index mismatch)
I20240501 08:11:34.537936 734 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 192, Last known committed idx: 190, Time since last communication: 0.000s
I20240501 08:11:34.540946 614 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:34.548328 32300 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:34.556677 30565 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:34.560586 31094 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:34.566358 30840 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:34.576102 31230 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:34.585850 31360 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 189. Preceding OpId from leader: term: 1 index: 192. (index mismatch)
I20240501 08:11:34.608994 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 192, Last known committed idx: 188, Time since last communication: 0.000s
I20240501 08:11:34.615195 31359 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.192: config changed from index 170 to 192, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 192 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:34.651645 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 54432.8us
I20240501 08:11:35.056151 30703 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 196, Committed index: 196, Last appended: 1.197, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
W20240501 08:11:35.076654 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 65864us
I20240501 08:11:35.140590 32301 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 197. Preceding OpId from leader: term: 1 index: 198. (index mismatch)
I20240501 08:11:35.141273 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 198, Last known committed idx: 196, Time since last communication: 0.000s
I20240501 08:11:35.142400 31099 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 197. Preceding OpId from leader: term: 1 index: 198. (index mismatch)
I20240501 08:11:35.145725 31227 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 197. Preceding OpId from leader: term: 1 index: 198. (index mismatch)
I20240501 08:11:35.147655 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 198, Last known committed idx: 196, Time since last communication: 0.000s
I20240501 08:11:35.150574 614 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 198, Last known committed idx: 196, Time since last communication: 0.000s
I20240501 08:11:35.157603 31354 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 197. Preceding OpId from leader: term: 1 index: 198. (index mismatch)
I20240501 08:11:35.157451 709 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.198: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:35.161880 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b3a5e3d4cc604fcd90a0efeea3946acd with cas_config_opid_index 192: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:35.162999 767 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 198, Last known committed idx: 196, Time since last communication: 0.003s
I20240501 08:11:35.165707 30564 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:35.173704 32301 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.198: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:35.192587 31098 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.198: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:35.197141 30818 tablet_service.cc:1514] Processing DeleteTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd with delete_type TABLET_DATA_TOMBSTONED (TS 1a388864321846929919a37b570e8138 not found in new config with opid_index 198) from {username='slave'} at 127.0.0.1:37872
I20240501 08:11:35.199929 31354 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.198: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:35.209244 31230 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.198: config changed from index 192 to 198, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 198 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:35.236646 776 tablet_replica.cc:330] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: stopping tablet replica
W20240501 08:11:35.260636 30906 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 63745.3us
I20240501 08:11:35.264477 776 raft_consensus.cc:2229] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:35.293681 776 pending_rounds.cc:62] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Trying to abort 1 pending ops.
I20240501 08:11:35.294065 776 pending_rounds.cc:68] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Aborting op as it isn't in flight: id { term: 1 index: 197 } timestamp: 7022801285278900224 op_type: WRITE_OP write_request { tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\034v\024x\240\0209\271\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""9ce1e73ae62780f3" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801284789350400 authz_token { token_data: "\010\316\364\307\261\006\"3\n\005slave\022*\n 9d6e5a2902824bacbbd466c9e3ece3b1\020\001\030\001 \001(\001" signature: "<redacted>""\3106D\002\215\023PA\335\343\016j\204\307\222\027!\244[\323\271W\002\245<\2712\370\036\334\0249rGX\325Z\227\252\364\337\230/Ct\255\022\204\020\026\363E\034\312\267\210R\350\001\036\326\241\021\021" signing_key_seq_num: 0 } } request_id { client_id: "131b712add9b42389c516d0752dd1023" seq_no: 591 first_incomplete_seq_no: 591 attempt_no: 0 }
I20240501 08:11:35.300215 776 raft_consensus.cc:2258] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:35.364739 776 ts_tablet_manager.cc:1875] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240501 08:11:35.416070 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 59155.2us
I20240501 08:11:35.566354 776 ts_tablet_manager.cc:1888] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.197
I20240501 08:11:35.566601 776 log.cc:1196] T b3a5e3d4cc604fcd90a0efeea3946acd P 1a388864321846929919a37b570e8138: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:11:35.567724 30555 catalog_manager.cc:4830] TS 1a388864321846929919a37b570e8138 (127.28.1.130:46291): tablet b3a5e3d4cc604fcd90a0efeea3946acd (table rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]) successfully deleted
W20240501 08:11:35.724460 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 45199.8us
W20240501 08:11:35.912173 596 consensus_queue.cc:1159] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "9204a82012ac492dbc0da9c84c51dabf" responder_term: 1 status { last_received { term: 1 index: 212 } last_committed_idx: 212 last_received_current_leader { term: 1 index: 212 } }
I20240501 08:11:35.924189 30967 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 212, Committed index: 212, Last appended: 1.212, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:35.941079 31357 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 212. Preceding OpId from leader: term: 1 index: 213. (index mismatch)
I20240501 08:11:35.941951 749 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 213, Last known committed idx: 212, Time since last communication: 0.000s
I20240501 08:11:35.942215 31099 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 212. Preceding OpId from leader: term: 1 index: 213. (index mismatch)
I20240501 08:11:35.942868 749 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 213, Last known committed idx: 212, Time since last communication: 0.000s
W20240501 08:11:35.945369 30905 consensus_peers.cc:485] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9ff3c78386224499b7022dc9334ddd33. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:35.945693 31227 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 212. Preceding OpId from leader: term: 1 index: 213. (index mismatch)
I20240501 08:11:35.946295 748 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 213, Last known committed idx: 212, Time since last communication: 0.000s
I20240501 08:11:35.947249 30703 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 212. Preceding OpId from leader: term: 1 index: 213. (index mismatch)
I20240501 08:11:35.949980 748 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.213: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:35.953419 31230 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.213: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:35.956037 749 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 213, Last known committed idx: 212, Time since last communication: 0.000s
I20240501 08:11:35.956534 31357 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.213: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:35.959647 30564 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:35.959718 31098 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.213: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:35.982163 30703 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.213: config changed from index -1 to 213, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:36.065819 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 166533us
W20240501 08:11:36.079018 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 156210us
W20240501 08:11:36.079567 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 176146us
W20240501 08:11:36.243757 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 99994us
I20240501 08:11:36.246204 30970 consensus_queue.cc:237] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 216, Committed index: 216, Last appended: 1.216, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
W20240501 08:11:36.252216 30905 consensus_peers.cc:485] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a465d27916734303bd9d1bed2a3dfb14. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:36.252377 30834 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 216. Preceding OpId from leader: term: 1 index: 217. (index mismatch)
I20240501 08:11:36.253046 777 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 217, Last known committed idx: 216, Time since last communication: 0.000s
I20240501 08:11:36.262703 30708 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 216. Preceding OpId from leader: term: 1 index: 217. (index mismatch)
I20240501 08:11:36.262671 31357 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 216. Preceding OpId from leader: term: 1 index: 217. (index mismatch)
I20240501 08:11:36.263621 596 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 217, Last known committed idx: 216, Time since last communication: 0.000s
I20240501 08:11:36.266544 748 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 217, Last known committed idx: 216, Time since last communication: 0.003s
I20240501 08:11:36.272266 366 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.217: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:36.275849 31357 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.217: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:36.279206 30703 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.217: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:36.291882 30840 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.217: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:36.294340 31094 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 216. Preceding OpId from leader: term: 1 index: 217. (index mismatch)
I20240501 08:11:36.305229 366 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 217, Last known committed idx: 216, Time since last communication: 0.000s
I20240501 08:11:36.313232 31099 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.217: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:36.379714 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 43559.4us
I20240501 08:11:36.541682 30566 catalog_manager.cc:5484] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index -1 to 217, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:36.623936 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 125232us
I20240501 08:11:36.859401 832 ts_tablet_manager.cc:902] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:36.967108 827 ts_tablet_manager.cc:902] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:11:37.164686 832 tablet_copy_client.cc:308] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:39451
I20240501 08:11:37.257241 30990 tablet_copy_service.cc:140] P 57210dc0a07a4851a43be23a279eb836: Received BeginTabletCopySession request for tablet 9ff3c78386224499b7022dc9334ddd33 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:57155)
I20240501 08:11:37.257431 30990 tablet_copy_service.cc:161] P 57210dc0a07a4851a43be23a279eb836: Beginning new tablet copy session on tablet 9ff3c78386224499b7022dc9334ddd33 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:57155: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-9ff3c78386224499b7022dc9334ddd33
I20240501 08:11:37.260876 30990 tablet_copy_source_session.cc:205] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:37.350315 832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:37.381358 832 tablet_copy_client.cc:783] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:37.381784 832 tablet_copy_client.cc:647] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:37.525247 832 tablet_copy_client.cc:523] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:37.543720 832 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:37.804958 31226 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 221, Committed index: 221, Last appended: 1.221, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
W20240501 08:11:37.809824 31165 consensus_peers.cc:485] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e61aa5646d3947c7b68bef19ee280c6e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:37.810052 31094 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 221. Preceding OpId from leader: term: 1 index: 222. (index mismatch)
I20240501 08:11:37.811034 822 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 222, Last known committed idx: 221, Time since last communication: 0.000s
I20240501 08:11:37.817293 30834 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 221. Preceding OpId from leader: term: 1 index: 222. (index mismatch)
I20240501 08:11:37.824270 822 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 222, Last known committed idx: 221, Time since last communication: 0.000s
I20240501 08:11:37.830212 822 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.222: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:37.831140 30834 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.222: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:37.837554 31094 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.222: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:37.840104 30560 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:37.861335 30703 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 221. Preceding OpId from leader: term: 1 index: 222. (index mismatch)
I20240501 08:11:37.896488 30970 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 220. Preceding OpId from leader: term: 1 index: 222. (index mismatch)
I20240501 08:11:37.900197 827 tablet_copy_client.cc:308] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:39451
I20240501 08:11:37.901612 30990 tablet_copy_service.cc:140] P 57210dc0a07a4851a43be23a279eb836: Received BeginTabletCopySession request for tablet a465d27916734303bd9d1bed2a3dfb14 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:57155)
I20240501 08:11:37.901811 30990 tablet_copy_service.cc:161] P 57210dc0a07a4851a43be23a279eb836: Beginning new tablet copy session on tablet a465d27916734303bd9d1bed2a3dfb14 from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:57155: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-a465d27916734303bd9d1bed2a3dfb14
I20240501 08:11:37.905042 30990 tablet_copy_source_session.cc:205] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:37.906934 827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a465d27916734303bd9d1bed2a3dfb14. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:37.912952 827 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.StartTabletCopy from 127.28.1.131:34027 (request call id 105) took 1097 ms. Trace:
I20240501 08:11:37.913128 827 rpcz_store.cc:276] 0501 08:11:36.815456 (+ 0us) service_pool.cc:168] Inserting onto call queue
0501 08:11:36.815516 (+ 60us) service_pool.cc:225] Handling call
0501 08:11:36.967300 (+151784us) ts_tablet_manager.cc:903] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
0501 08:11:37.906359 (+939059us) tablet_copy_client.cc:353] Tablet copy session begun
0501 08:11:37.910038 (+ 3679us) tablet_metadata.cc:641] Metadata flushed
0501 08:11:37.910179 (+ 141us) tablet_copy_client.cc:482] Wrote new tablet metadata
0501 08:11:37.912632 (+ 2453us) tablet_copy_client.cc:403] Wrote new consensus metadata
0501 08:11:37.912788 (+ 156us) tablet_replica.cc:205] Creating consensus instance
0501 08:11:37.912881 (+ 93us) ts_tablet_manager.cc:958] Replica registered.
0501 08:11:37.912907 (+ 26us) inbound_call.cc:173] Queueing success response
Metrics: {"dns_us":93,"tablet-copy.queue_time_us":151337,"thread_start_us":976013,"threads_started":5}
I20240501 08:11:37.914541 827 tablet_copy_client.cc:783] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:37.914871 827 tablet_copy_client.cc:647] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:37.950644 827 tablet_copy_client.cc:523] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:37.954895 827 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
W20240501 08:11:38.087302 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 168070us
W20240501 08:11:38.319257 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 393051us
I20240501 08:11:38.344098 884 ts_tablet_manager.cc:902] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:11:38.360040 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 222, Last known committed idx: 220, Time since last communication: 0.000s
I20240501 08:11:38.384405 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 222, Last known committed idx: 220, Time since last communication: 0.000s
I20240501 08:11:38.394271 30706 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.222: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:38.691908 30967 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.222: config changed from index -1 to 222, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:38.761232 30834 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 159, Committed index: 159, Last appended: 1.159, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:38.784986 31099 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:38.788728 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 159, Time since last communication: 0.000s
I20240501 08:11:38.790917 30709 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
W20240501 08:11:38.791846 32297 thread.cc:636] prepare [worker] (thread pool) Time spent creating pthread: real 0.548s user 0.007s sys 0.078s
W20240501 08:11:38.792025 32297 thread.cc:602] prepare [worker] (thread pool) Time spent starting thread: real 0.548s user 0.007s sys 0.078s
I20240501 08:11:38.800887 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 159, Time since last communication: 0.000s
I20240501 08:11:38.824677 31359 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
I20240501 08:11:38.826009 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 159, Time since last communication: 0.000s
I20240501 08:11:38.829468 764 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.160: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:38.838321 31359 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:38.842248 30775 consensus_peers.cc:485] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 568f82bc574e4c6fafe9ddb3375429ae. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:11:38.846447 30567 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:38.846670 31093 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:38.857353 30708 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
I20240501 08:11:38.877512 31227 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 160. (index mismatch)
W20240501 08:11:38.978474 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 97824.7us
I20240501 08:11:38.982306 830 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 160, Last known committed idx: 159, Time since last communication: 0.000s
I20240501 08:11:38.996007 31226 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.160: config changed from index -1 to 160, NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) added. New config: { opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } } }
W20240501 08:11:39.005128 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 124086us
I20240501 08:11:39.089016 884 tablet_copy_client.cc:308] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:11:39.093724 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet e61aa5646d3947c7b68bef19ee280c6e from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:35117)
I20240501 08:11:39.093932 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet e61aa5646d3947c7b68bef19ee280c6e from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:35117: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:11:39.096877 31250 tablet_copy_source_session.cc:205] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:39.098376 884 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:39.190085 884 tablet_copy_client.cc:783] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:39.211081 884 tablet_copy_client.cc:647] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
I20240501 08:11:39.218370 884 tablet_copy_client.cc:523] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:39.223665 884 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:39.547485 934 ts_tablet_manager.cc:902] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Initiating tablet copy from peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
W20240501 08:11:39.686612 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 173064us
W20240501 08:11:39.823695 832 log.cc:924] Time spent T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.238s user 0.001s sys 0.000s
I20240501 08:11:39.850682 934 tablet_copy_client.cc:308] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:46291
I20240501 08:11:39.938628 884 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=232 overwritten=0 applied=232 ignored=0} inserts{seen=904 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:39.943611 884 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:39.951453 884 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 0.728s user 0.264s sys 0.037s
I20240501 08:11:39.964267 884 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:39.964727 884 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:39.965003 884 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 232, Last appended: 1.232, Last appended by leader: 232, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 222 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:39.974547 884 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.023s user 0.002s sys 0.000s
I20240501 08:11:39.977335 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-e61aa5646d3947c7b68bef19ee280c6e received from {username='slave'} at 127.28.1.135:35117
I20240501 08:11:39.977483 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-e61aa5646d3947c7b68bef19ee280c6e on tablet e61aa5646d3947c7b68bef19ee280c6e with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:39.984037 32300 raft_consensus.cc:1203] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.221->[1.222-1.241] Dedup: 1.232->[1.233-1.241]
I20240501 08:11:40.194070 30860 tablet_copy_service.cc:140] P 1a388864321846929919a37b570e8138: Received BeginTabletCopySession request for tablet 568f82bc574e4c6fafe9ddb3375429ae from peer fe99ab4507a24af4ba7fa5b5278c5ac6 ({username='slave'} at 127.28.1.135:59899)
I20240501 08:11:40.194365 30860 tablet_copy_service.cc:161] P 1a388864321846929919a37b570e8138: Beginning new tablet copy session on tablet 568f82bc574e4c6fafe9ddb3375429ae from peer fe99ab4507a24af4ba7fa5b5278c5ac6 at {username='slave'} at 127.28.1.135:59899: session id = fe99ab4507a24af4ba7fa5b5278c5ac6-568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:11:40.200467 30860 tablet_copy_source_session.cc:205] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:11:40.215749 934 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:11:40.275660 934 tablet_copy_client.cc:783] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 0 data blocks...
I20240501 08:11:40.276053 934 tablet_copy_client.cc:647] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Starting download of 1 WAL segments...
W20240501 08:11:40.327359 30905 consensus_peers.cc:485] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 -> Peer fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): Couldn't send request to peer fe99ab4507a24af4ba7fa5b5278c5ac6. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240501 08:11:40.327631 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 71160us
I20240501 08:11:40.347848 815 raft_consensus.cc:1050] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:40.348852 815 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 247, Committed index: 247, Last appended: 1.247, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:40.353837 32299 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:40.354511 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:40.357868 30703 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:40.359709 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:40.360273 31093 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:40.360855 903 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:40.365409 30834 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:40.368381 903 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:40.376627 32494 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.380007 832 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=222 overwritten=0 applied=222 ignored=0} inserts{seen=907 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:40.380096 30967 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:40.385818 31099 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.388130 30709 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.389667 30566 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:40.401808 30834 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.413187 934 tablet_copy_client.cc:523] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:11:40.445688 934 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap starting.
I20240501 08:11:40.453186 903 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:40.390833 32300 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.462456 832 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:40.468955 832 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 2.925s user 0.311s sys 0.019s
I20240501 08:11:40.470278 832 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
W20240501 08:11:40.502027 30906 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 48066.3us
I20240501 08:11:40.541689 30967 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 222 to 248, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.550752 832 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:40.551105 832 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 222, Last appended: 1.222, Last appended by leader: 222, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 213 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:40.552217 832 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.083s user 0.002s sys 0.000s
I20240501 08:11:40.579710 30990 tablet_copy_service.cc:342] P 57210dc0a07a4851a43be23a279eb836: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-9ff3c78386224499b7022dc9334ddd33 received from {username='slave'} at 127.28.1.135:57155
I20240501 08:11:40.579869 30990 tablet_copy_service.cc:433] P 57210dc0a07a4851a43be23a279eb836: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-9ff3c78386224499b7022dc9334ddd33 on tablet 9ff3c78386224499b7022dc9334ddd33 with peer fe99ab4507a24af4ba7fa5b5278c5ac6
W20240501 08:11:40.637559 31293 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 43915.4us
I20240501 08:11:40.719266 32300 raft_consensus.cc:1203] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.212->[1.213-1.244] Dedup: 1.222->[1.223-1.244]
I20240501 08:11:40.808269 827 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=223 overwritten=0 applied=223 ignored=0} inserts{seen=908 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:40.814841 827 tablet_bootstrap.cc:492] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:40.815282 827 ts_tablet_manager.cc:1367] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 2.861s user 0.304s sys 0.019s
I20240501 08:11:40.816241 827 raft_consensus.cc:348] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:40.823379 827 raft_consensus.cc:729] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:40.823760 827 consensus_queue.cc:260] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 223, Last appended: 1.223, Last appended by leader: 223, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 217 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:40.824553 827 ts_tablet_manager.cc:1398] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240501 08:11:40.835060 31228 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 249, Committed index: 249, Last appended: 1.249, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:40.855445 32299 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 249. Preceding OpId from leader: term: 1 index: 251. (index mismatch)
I20240501 08:11:40.856068 30709 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 249. Preceding OpId from leader: term: 1 index: 251. (index mismatch)
I20240501 08:11:40.856153 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 250, Last known committed idx: 249, Time since last communication: 0.000s
I20240501 08:11:40.857465 32494 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 250, Last known committed idx: 249, Time since last communication: 0.000s
I20240501 08:11:40.859405 30967 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 249. Preceding OpId from leader: term: 1 index: 251. (index mismatch)
I20240501 08:11:40.860396 31096 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 249. Preceding OpId from leader: term: 1 index: 251. (index mismatch)
I20240501 08:11:40.861488 904 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 250, Last known committed idx: 248, Time since last communication: 0.000s
I20240501 08:11:40.862064 904 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 250, Last known committed idx: 249, Time since last communication: 0.000s
I20240501 08:11:40.862455 954 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.250: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.864215 31094 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.250: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.871614 32301 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.250: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.867702 30708 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.250: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:40.883071 30642 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 287093us
I20240501 08:11:40.889107 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e61aa5646d3947c7b68bef19ee280c6e with cas_config_opid_index 248: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:40.898178 30567 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:40.905828 30967 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.250: config changed from index 248 to 250, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 250 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:40.918870 30818 tablet_service.cc:1514] Processing DeleteTablet for tablet e61aa5646d3947c7b68bef19ee280c6e with delete_type TABLET_DATA_TOMBSTONED (TS 1a388864321846929919a37b570e8138 not found in new config with opid_index 250) from {username='slave'} at 127.0.0.1:37872
I20240501 08:11:40.919611 995 tablet_replica.cc:330] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: stopping tablet replica
I20240501 08:11:40.920267 995 raft_consensus.cc:2229] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:40.920478 995 pending_rounds.cc:62] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Trying to abort 1 pending ops.
I20240501 08:11:40.920543 995 pending_rounds.cc:68] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Aborting op as it isn't in flight: id { term: 1 index: 249 } timestamp: 7022801308976754688 op_type: WRITE_OP write_request { tablet_id: "e61aa5646d3947c7b68bef19ee280c6e" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\206\257Ed\304!\223V\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""6db23dcb21ede3ba" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801308807999488 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n e93d8483bead454d8e47e99f73d1dabb\020\001\030\001 \001(\001" signature: "<redacted>""\272O\355\373\2120\201?\215\020O\361\323\366\223 f|\327o\347\004\254o\005%\033\304\262v\207\215\312Y\301gu\262\306\255O,a\035\247\306\033=\200.\257\262\277\006v,\233\224\003\370\247\330w\212" signing_key_seq_num: 0 } } request_id { client_id: "c52c7e7b69284539914688f45a543b0c" seq_no: 745 first_incomplete_seq_no: 745 attempt_no: 0 }
I20240501 08:11:40.921298 995 raft_consensus.cc:2258] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:40.923233 995 ts_tablet_manager.cc:1875] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:40.936508 30990 tablet_copy_service.cc:342] P 57210dc0a07a4851a43be23a279eb836: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-a465d27916734303bd9d1bed2a3dfb14 received from {username='slave'} at 127.28.1.135:57155
I20240501 08:11:40.936662 30990 tablet_copy_service.cc:433] P 57210dc0a07a4851a43be23a279eb836: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-a465d27916734303bd9d1bed2a3dfb14 on tablet a465d27916734303bd9d1bed2a3dfb14 with peer fe99ab4507a24af4ba7fa5b5278c5ac6
W20240501 08:11:40.938565 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 55562.5us
I20240501 08:11:40.944993 995 ts_tablet_manager.cc:1888] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.249
I20240501 08:11:40.948204 995 log.cc:1196] T e61aa5646d3947c7b68bef19ee280c6e P 1a388864321846929919a37b570e8138: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:11:40.949419 30555 catalog_manager.cc:4830] TS 1a388864321846929919a37b570e8138 (127.28.1.130:46291): tablet e61aa5646d3947c7b68bef19ee280c6e (table rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]) successfully deleted
W20240501 08:11:41.004700 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 173275us
W20240501 08:11:41.036430 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 96003.7us
I20240501 08:11:41.086755 940 raft_consensus.cc:1050] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:41.095822 940 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 248, Committed index: 248, Last appended: 1.248, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:41.102922 30706 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 248. Preceding OpId from leader: term: 1 index: 249. (index mismatch)
I20240501 08:11:41.103566 915 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 249, Last known committed idx: 248, Time since last communication: 0.000s
I20240501 08:11:41.108950 31095 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 248. Preceding OpId from leader: term: 1 index: 249. (index mismatch)
I20240501 08:11:41.109848 31228 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 248. Preceding OpId from leader: term: 1 index: 249. (index mismatch)
I20240501 08:11:41.136363 31360 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 248. Preceding OpId from leader: term: 1 index: 249. (index mismatch)
I20240501 08:11:41.136643 32299 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 248. Preceding OpId from leader: term: 1 index: 249. (index mismatch)
I20240501 08:11:41.138564 854 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 249, Last known committed idx: 248, Time since last communication: 0.001s
I20240501 08:11:41.162796 32301 raft_consensus.cc:1203] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.216->[1.217-1.245] Dedup: 1.223->[1.224-1.245]
I20240501 08:11:41.205917 854 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 249, Last known committed idx: 248, Time since last communication: 0.000s
I20240501 08:11:41.238708 854 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 249, Last known committed idx: 248, Time since last communication: 0.000s
I20240501 08:11:41.278862 854 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:41.286106 1002 log.cc:924] Time spent T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6: Append to log took a long time: real 0.083s user 0.000s sys 0.000s
I20240501 08:11:41.288360 30706 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.294313 30567 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 reported cstate change: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:41.296134 31360 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.313802 32301 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.439261 915 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 249, Last known committed idx: 248, Time since last communication: 0.000s
W20240501 08:11:41.469447 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 70593.3us
I20240501 08:11:41.500559 31095 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.519814 31230 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.249: config changed from index 213 to 249, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.690582 748 raft_consensus.cc:1050] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
W20240501 08:11:41.691419 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 91454.5us
I20240501 08:11:41.691504 748 consensus_queue.cc:237] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 246, Committed index: 246, Last appended: 1.246, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
W20240501 08:11:41.692401 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 86850.3us
I20240501 08:11:41.708513 30703 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240501 08:11:41.709022 32298 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240501 08:11:41.709731 31359 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240501 08:11:41.713521 777 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240501 08:11:41.714066 777 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 245, Time since last communication: 0.000s
I20240501 08:11:41.716763 1007 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240501 08:11:41.720067 31100 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240501 08:11:41.723409 854 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 245, Time since last communication: 0.000s
I20240501 08:11:41.748888 30837 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240501 08:11:41.752946 854 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.756461 940 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240501 08:11:41.757448 31100 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.777776 30567 catalog_manager.cc:5484] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:41.806311 30837 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:41.833920 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 50575.4us
I20240501 08:11:41.838953 32298 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.008553 30703 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.019770 30970 consensus_queue.cc:237] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 247, Committed index: 247, Last appended: 1.247, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:42.023389 30834 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:42.024294 31093 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:42.024374 940 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:42.024984 940 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:42.028350 940 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.248: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.033668 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a465d27916734303bd9d1bed2a3dfb14 with cas_config_opid_index 247: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:42.035214 30834 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.043143 31360 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 217 to 247, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.051714 30567 catalog_manager.cc:5484] T a465d27916734303bd9d1bed2a3dfb14 P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.057963 30687 tablet_service.cc:1514] Processing DeleteTablet for tablet a465d27916734303bd9d1bed2a3dfb14 with delete_type TABLET_DATA_TOMBSTONED (TS 3170be3b2bbb4b3bbd8a37ac5f91de89 not found in new config with opid_index 248) from {username='slave'} at 127.0.0.1:43428
I20240501 08:11:42.057958 31100 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.064244 1026 tablet_replica.cc:330] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: stopping tablet replica
I20240501 08:11:42.064968 32301 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:42.066064 31360 raft_consensus.cc:1261] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 247. Preceding OpId from leader: term: 1 index: 248. (index mismatch)
I20240501 08:11:42.077237 1026 raft_consensus.cc:2229] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:42.077742 1026 raft_consensus.cc:2258] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:42.092783 1026 ts_tablet_manager.cc:1875] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:42.103169 915 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:42.149838 940 consensus_queue.cc:1035] T a465d27916734303bd9d1bed2a3dfb14 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 248, Last known committed idx: 247, Time since last communication: 0.000s
I20240501 08:11:42.152273 32298 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.171089 1026 ts_tablet_manager.cc:1888] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.247
I20240501 08:11:42.176419 31354 raft_consensus.cc:2932] T a465d27916734303bd9d1bed2a3dfb14 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.248: config changed from index 247 to 248, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.182622 1026 log.cc:1196] T a465d27916734303bd9d1bed2a3dfb14 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/a465d27916734303bd9d1bed2a3dfb14
I20240501 08:11:42.188975 30552 catalog_manager.cc:4830] TS 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): tablet a465d27916734303bd9d1bed2a3dfb14 (table rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]) successfully deleted
I20240501 08:11:42.197399 934 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap replayed 1/1 log segments. Stats: ops{read=177 overwritten=0 applied=177 ignored=0} inserts{seen=849 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:11:42.198046 934 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Bootstrap complete.
I20240501 08:11:42.198529 934 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent bootstrapping tablet: real 1.753s user 0.230s sys 0.023s
I20240501 08:11:42.199591 934 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:42.200109 934 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Initialized, Role: LEARNER
I20240501 08:11:42.200430 934 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 177, Last appended: 1.177, Last appended by leader: 177, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 160 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: true } }
I20240501 08:11:42.204277 934 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:11:42.208096 30860 tablet_copy_service.cc:342] P 1a388864321846929919a37b570e8138: Request end of tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-568f82bc574e4c6fafe9ddb3375429ae received from {username='slave'} at 127.28.1.135:59899
I20240501 08:11:42.208341 30860 tablet_copy_service.cc:433] P 1a388864321846929919a37b570e8138: ending tablet copy session fe99ab4507a24af4ba7fa5b5278c5ac6-568f82bc574e4c6fafe9ddb3375429ae on tablet 568f82bc574e4c6fafe9ddb3375429ae with peer fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:11:42.479400 32298 raft_consensus.cc:1203] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.159->[1.160-1.197] Dedup: 1.177->[1.178-1.197]
I20240501 08:11:42.763319 1015 raft_consensus.cc:1050] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: attempting to promote NON_VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 to VOTER
I20240501 08:11:42.787501 1015 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 200, Committed index: 200, Last appended: 1.200, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:42.791381 32298 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 LEARNER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 200. Preceding OpId from leader: term: 1 index: 201. (index mismatch)
I20240501 08:11:42.792176 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 201, Last known committed idx: 200, Time since last communication: 0.000s
I20240501 08:11:42.793901 30709 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 200. Preceding OpId from leader: term: 1 index: 201. (index mismatch)
I20240501 08:11:42.797020 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 201, Last known committed idx: 200, Time since last communication: 0.000s
I20240501 08:11:42.804497 31230 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 200. Preceding OpId from leader: term: 1 index: 201. (index mismatch)
I20240501 08:11:42.807739 31099 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 200. Preceding OpId from leader: term: 1 index: 201. (index mismatch)
I20240501 08:11:42.809746 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 201, Last known committed idx: 199, Time since last communication: 0.000s
I20240501 08:11:42.810925 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 201, Last known committed idx: 199, Time since last communication: 0.000s
I20240501 08:11:42.817893 1015 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.826157 31098 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.887375 32299 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.892594 30567 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:42.888980 30703 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:42.917179 31230 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.012327 31360 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 200. Preceding OpId from leader: term: 1 index: 201. (index mismatch)
I20240501 08:11:43.120673 830 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 201, Last known committed idx: 199, Time since last communication: 0.000s
I20240501 08:11:43.178131 31359 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.201: config changed from index 160 to 201, fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 201 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.238497 30837 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 202, Committed index: 202, Last appended: 1.203, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:43.246026 32298 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 203. Preceding OpId from leader: term: 1 index: 204. (index mismatch)
I20240501 08:11:43.247263 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 204, Last known committed idx: 202, Time since last communication: 0.000s
I20240501 08:11:43.264815 31095 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 203. Preceding OpId from leader: term: 1 index: 204. (index mismatch)
I20240501 08:11:43.271179 31352 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 202. Preceding OpId from leader: term: 1 index: 204. (index mismatch)
I20240501 08:11:43.278950 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 204, Last known committed idx: 202, Time since last communication: 0.000s
I20240501 08:11:43.298545 693 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 204, Last known committed idx: 202, Time since last communication: 0.000s
I20240501 08:11:43.311815 31226 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 202. Preceding OpId from leader: term: 1 index: 204. (index mismatch)
I20240501 08:11:43.317312 830 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.204: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.321249 31352 raft_consensus.cc:1203] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.202->[1.203-1.204] Dedup: 1.203->[1.204-1.204]
I20240501 08:11:43.325945 30555 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 568f82bc574e4c6fafe9ddb3375429ae with cas_config_opid_index 201: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:43.331657 32299 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.204: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.331718 31098 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.204: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.332111 30567 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:11:43.336370 30687 tablet_service.cc:1514] Processing DeleteTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae with delete_type TABLET_DATA_TOMBSTONED (TS 3170be3b2bbb4b3bbd8a37ac5f91de89 not found in new config with opid_index 204) from {username='slave'} at 127.0.0.1:43428
I20240501 08:11:43.395556 1064 tablet_replica.cc:330] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: stopping tablet replica
I20240501 08:11:43.395808 1064 raft_consensus.cc:2229] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:43.397238 1064 pending_rounds.cc:62] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Trying to abort 1 pending ops.
I20240501 08:11:43.397327 1064 pending_rounds.cc:68] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Aborting op as it isn't in flight: id { term: 1 index: 203 } timestamp: 7022801318806523904 op_type: WRITE_OP write_request { tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000R\014\272w\352\320;Y\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""aefec1bfaec4370e" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7022801318366212096 authz_token { token_data: "\010\315\364\307\261\006\"3\n\005slave\022*\n 611625662521483486949c0824ba78a9\020\001\030\001 \001(\001" signature: "<redacted>""\203\314\t\036E(\022_\016Tkx\026t9^>\"W\311=!\021\007t\0204IjK<\257\334\0044\320W\332w\261\230\252\036\"\206\236;\001\025\007\307\276u\277)fJs\253N\354{\ra" signing_key_seq_num: 0 } } request_id { client_id: "74e6484188604f04b687ec475c11592e" seq_no: 617 first_incomplete_seq_no: 617 attempt_no: 0 }
I20240501 08:11:43.397912 1064 raft_consensus.cc:2258] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:11:43.418262 31352 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.204: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.426558 830 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 204, Last known committed idx: 201, Time since last communication: 0.000s
I20240501 08:11:43.451722 31226 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.204: config changed from index 201 to 204, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 204 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:43.432582 1064 ts_tablet_manager.cc:1875] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:43.605660 1064 ts_tablet_manager.cc:1888] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.203
I20240501 08:11:43.605921 1064 log.cc:1196] T 568f82bc574e4c6fafe9ddb3375429ae P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:11:43.607934 30552 catalog_manager.cc:4830] TS 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): tablet 568f82bc574e4c6fafe9ddb3375429ae (table rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]) successfully deleted
W20240501 08:11:43.974552 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 78085.8us
I20240501 08:11:44.041491 30966 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33"
dest_uuid: "57210dc0a07a4851a43be23a279eb836"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:35410
I20240501 08:11:44.041822 30966 raft_consensus.cc:595] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Received request to transfer leadership
W20240501 08:11:44.077844 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 75166.7us
W20240501 08:11:44.100281 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.140064 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.171880 30930 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.190127 30930 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.198762 30930 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.210938 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
I20240501 08:11:44.284765 1019 raft_consensus.cc:979] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: : Instructing follower 9fc0640dbc0d4208bfb536949d1bb0c4 to start an election
I20240501 08:11:44.284958 1019 raft_consensus.cc:1067] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Signalling peer 9fc0640dbc0d4208bfb536949d1bb0c4 to start an election
I20240501 08:11:44.286131 31093 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33"
dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
from {username='slave'} at 127.28.1.131:37451
I20240501 08:11:44.288450 31093 raft_consensus.cc:482] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
W20240501 08:11:44.291199 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
I20240501 08:11:44.294178 31093 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Advancing to term 2
W20240501 08:11:44.305667 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.318928 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
W20240501 08:11:44.344323 30927 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Service unavailable: leader transfer in progress
I20240501 08:11:44.356880 31093 raft_consensus.cc:504] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:44.363188 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 2 candidate_status { last_received { term: 1 index: 270 } } ignore_live_leader: true dest_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89"
I20240501 08:11:44.363523 30703 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:11:44.367331 31228 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 2 candidate_status { last_received { term: 1 index: 270 } } ignore_live_leader: true dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:11:44.367426 30703 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 2.
I20240501 08:11:44.367638 31228 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:11:44.370733 31228 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 2.
I20240501 08:11:44.379436 32297 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 2 candidate_status { last_received { term: 1 index: 270 } } ignore_live_leader: true dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
I20240501 08:11:44.379813 32297 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:11:44.383713 32297 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 2.
I20240501 08:11:44.386049 32299 raft_consensus.cc:1226] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Rejecting Update request from peer 57210dc0a07a4851a43be23a279eb836 for earlier term 1. Current term is 2. Ops: []
I20240501 08:11:44.388202 31359 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 2 candidate_status { last_received { term: 1 index: 270 } } ignore_live_leader: true dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:11:44.388510 31359 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:11:44.391268 31359 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 2.
I20240501 08:11:44.395184 31035 leader_election.cc:302] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 6 voters: 4 yes votes; 0 no votes. yes voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4; no voters:
I20240501 08:11:44.395613 1009 raft_consensus.cc:2781] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:11:44.395764 1009 raft_consensus.cc:686] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 LEADER]: Becoming Leader. State: Replica: 9fc0640dbc0d4208bfb536949d1bb0c4, State: Running, Role: LEADER
I20240501 08:11:44.396669 1009 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 270, Committed index: 270, Last appended: 1.270, Last appended by leader: 270, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:44.401459 30567 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: term changed from 1 to 2, leader changed from 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) to 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132). New cstate: current_term: 2 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:44.363310 30968 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ff3c78386224499b7022dc9334ddd33" candidate_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" candidate_term: 2 candidate_status { last_received { term: 1 index: 270 } } ignore_live_leader: true dest_uuid: "57210dc0a07a4851a43be23a279eb836"
I20240501 08:11:44.386646 748 consensus_queue.cc:1046] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: INVALID_TERM, Last received: 1.270, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240501 08:11:44.403079 30968 raft_consensus.cc:3032] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:11:44.403192 30968 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 57210dc0a07a4851a43be23a279eb836, State: Running, Role: LEADER
I20240501 08:11:44.403537 30968 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 270, Committed index: 270, Last appended: 1.270, Last appended by leader: 270, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 249 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:44.404212 31093 leader_election.cc:288] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [CANDIDATE]: Term 2 election: Requested vote from peers 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533)
I20240501 08:11:44.404676 30968 raft_consensus.cc:3037] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:11:44.408947 30968 raft_consensus.cc:2454] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9fc0640dbc0d4208bfb536949d1bb0c4 in term 2.
W20240501 08:11:44.436507 30930 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:46676: Illegal state: replica 57210dc0a07a4851a43be23a279eb836 is not leader of this config: current role FOLLOWER
I20240501 08:11:44.846499 30968 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 2 index: 271. (index mismatch)
I20240501 08:11:44.847617 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240501 08:11:44.877249 31360 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 2 index: 271. (index mismatch)
I20240501 08:11:44.877913 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
W20240501 08:11:44.892910 31190 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:37074: Illegal state: replica 9204a82012ac492dbc0da9c84c51dabf is not leader of this config: current role FOLLOWER
I20240501 08:11:44.906373 32299 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 2 index: 271. (index mismatch)
I20240501 08:11:44.907510 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240501 08:11:44.908367 30706 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 2 index: 271. (index mismatch)
I20240501 08:11:44.909372 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240501 08:11:44.969349 31230 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 2 index: 271. (index mismatch)
I20240501 08:11:44.970048 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240501 08:11:45.398933 31099 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 272, Committed index: 272, Last appended: 2.272, Last appended by leader: 270, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:11:45.403326 31360 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 272. Preceding OpId from leader: term: 2 index: 273. (index mismatch)
I20240501 08:11:45.404651 30707 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 272. Preceding OpId from leader: term: 2 index: 273. (index mismatch)
I20240501 08:11:45.404645 1036 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 273, Last known committed idx: 272, Time since last communication: 0.000s
I20240501 08:11:45.410482 1009 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 273, Last known committed idx: 272, Time since last communication: 0.000s
I20240501 08:11:45.418212 1036 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 LEADER]: Committing config change with OpId 2.273: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:45.419070 32293 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 272. Preceding OpId from leader: term: 2 index: 273. (index mismatch)
I20240501 08:11:45.420169 1041 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 273, Last known committed idx: 272, Time since last communication: 0.000s
I20240501 08:11:45.421725 32295 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Committing config change with OpId 2.273: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:45.423995 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9ff3c78386224499b7022dc9334ddd33 with cas_config_opid_index 249: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:11:45.425335 30708 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Committing config change with OpId 2.273: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:45.426062 30567 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New cstate: current_term: 2 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:11:45.426585 31228 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 272. Preceding OpId from leader: term: 2 index: 273. (index mismatch)
I20240501 08:11:45.427165 1036 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 273, Last known committed idx: 272, Time since last communication: 0.000s
I20240501 08:11:45.428426 31360 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Committing config change with OpId 2.273: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
I20240501 08:11:45.428521 31226 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Committing config change with OpId 2.273: config changed from index 249 to 273, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 273 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } }
W20240501 08:11:45.773885 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 66813.5us
W20240501 08:11:45.880940 30644 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 165628us
I20240501 08:11:45.932252 30948 tablet_service.cc:1514] Processing DeleteTablet for tablet 9ff3c78386224499b7022dc9334ddd33 with delete_type TABLET_DATA_TOMBSTONED (TS 57210dc0a07a4851a43be23a279eb836 not found in new config with opid_index 273) from {username='slave'} at 127.0.0.1:44190
I20240501 08:11:45.962544 1178 tablet_replica.cc:330] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: stopping tablet replica
I20240501 08:11:45.962785 1178 raft_consensus.cc:2229] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:11:45.988230 1178 raft_consensus.cc:2258] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836 [term 2 FOLLOWER]: Raft consensus is shut down!
W20240501 08:11:46.003572 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 60521.2us
W20240501 08:11:46.013233 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 82767.1us
W20240501 08:11:46.020658 30644 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 60007.8us
I20240501 08:11:46.034250 1178 ts_tablet_manager.cc:1875] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:11:46.076412 1178 ts_tablet_manager.cc:1888] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.272
I20240501 08:11:46.091922 1178 log.cc:1196] T 9ff3c78386224499b7022dc9334ddd33 P 57210dc0a07a4851a43be23a279eb836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/9ff3c78386224499b7022dc9334ddd33
W20240501 08:11:46.092492 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 152903us
I20240501 08:11:46.093070 30552 catalog_manager.cc:4830] TS 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): tablet 9ff3c78386224499b7022dc9334ddd33 (table rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]) successfully deleted
W20240501 08:11:46.213793 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 97502.7us
W20240501 08:11:46.325609 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 88519us
W20240501 08:11:46.333864 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 92381.7us
W20240501 08:11:46.404124 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 54330.4us
W20240501 08:11:46.411353 31164 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 57497.7us
W20240501 08:11:46.411825 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 57703.4us
W20240501 08:11:46.412075 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 57698.6us
W20240501 08:11:46.411825 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 60656.5us
W20240501 08:11:46.629005 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 218771us
W20240501 08:11:46.878312 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 97086.7us
W20240501 08:11:46.949595 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 103944us
W20240501 08:11:47.031831 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 55778.9us
W20240501 08:11:47.356431 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 43875.1us
I20240501 08:11:47.754607 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:47.901011 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:47.922514 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:47.958411 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240501 08:11:47.964723 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 59245us
W20240501 08:11:48.032147 30903 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 54897.3us
W20240501 08:11:48.034119 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 57221us
W20240501 08:11:48.040066 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 61054.9us
W20240501 08:11:48.057056 30906 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 79521.3us
W20240501 08:11:48.100055 30643 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 52883.4us
I20240501 08:11:48.119663 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:48.301676 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240501 08:11:48.374748 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 83911.6us
I20240501 08:11:48.443302 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240501 08:11:50.040966 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 57899.4us
W20240501 08:11:50.731557 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 72792.8us
W20240501 08:11:50.843112 31293 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 89489.5us
W20240501 08:11:51.096151 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 111214us
W20240501 08:11:51.166152 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 61421.3us
W20240501 08:11:51.470901 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 45833.7us
I20240501 08:11:51.494263 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:51.532039 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:51.570350 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:11:51.631244 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:51.639266 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:51.693166 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:11:51.968367 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240501 08:11:52.056159 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 59903.5us
W20240501 08:11:52.169862 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 174000us
W20240501 08:11:52.310914 30906 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 114570us
W20240501 08:11:52.312829 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 113345us
W20240501 08:11:52.825361 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 58312.5us
W20240501 08:11:53.471666 31163 outbound_call.cc:322] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.28.1.190:35983, user_credentials={real_user=slave}} blocked reactor thread for 56506.8us
W20240501 08:11:53.615491 31165 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:41579, user_credentials={real_user=slave}} blocked reactor thread for 44143.8us
W20240501 08:11:53.733280 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 170194us
W20240501 08:11:53.958751 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 51147.9us
W20240501 08:11:54.062003 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 65364.7us
W20240501 08:11:54.422259 31034 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 70779.5us
W20240501 08:11:54.810452 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 80035.9us
W20240501 08:11:55.344020 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 58395.1us
I20240501 08:11:56.155714 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:11:56.192389 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:11:56.233759 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:11:56.284744 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:11:56.634465 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:11:56.811272 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240501 08:11:56.860235 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 45379.6us
I20240501 08:11:57.208961 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240501 08:11:57.845486 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 64751.4us
W20240501 08:11:57.892786 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 46777.7us
W20240501 08:11:58.213522 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 71721.8us
W20240501 08:11:58.404800 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 63904.2us
W20240501 08:11:59.376873 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 115947us
W20240501 08:11:59.446641 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 117405us
W20240501 08:11:59.567028 31296 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 93684.7us
I20240501 08:11:59.649462 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240501 08:11:59.823076 31164 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 341428us
W20240501 08:11:59.850268 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 52908.7us
W20240501 08:11:59.887063 30905 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:36533, user_credentials={real_user=slave}} blocked reactor thread for 45744.9us
W20240501 08:11:59.901362 30904 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42523, user_credentials={real_user=slave}} blocked reactor thread for 68711.9us
I20240501 08:11:59.913405 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240501 08:11:59.930117 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 56429.6us
I20240501 08:12:00.047205 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:00.297371 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:00.448590 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:00.476689 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240501 08:12:00.510197 31033 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 94833.1us
I20240501 08:12:00.832458 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240501 08:12:01.144904 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 62770.4us
W20240501 08:12:01.254745 30773 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:39451, user_credentials={real_user=slave}} blocked reactor thread for 48363.7us
W20240501 08:12:01.446202 31190 thread.cc:636] prepare [worker] (thread pool) Time spent creating pthread: real 0.505s user 0.016s sys 0.011s
W20240501 08:12:01.446409 31190 thread.cc:602] prepare [worker] (thread pool) Time spent starting thread: real 0.505s user 0.016s sys 0.011s
W20240501 08:12:01.703429 30774 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:41135, user_credentials={real_user=slave}} blocked reactor thread for 90589.7us
W20240501 08:12:01.857843 31166 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:46291, user_credentials={real_user=slave}} blocked reactor thread for 58515us
W20240501 08:12:01.864053 31163 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:39637, user_credentials={real_user=slave}} blocked reactor thread for 74046.2us
I20240501 08:12:02.249073 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:02.255779 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:02.258968 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:02.273300 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:02.281268 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:02.296533 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:02.298645 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
b5013b5f37d84aa1810fafbf49ae9861 | 127.28.1.190:35983 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:41967 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
1a388864321846929919a37b570e8138 | 127.28.1.130:46291 | HEALTHY | <none> | 3 | 0
3170be3b2bbb4b3bbd8a37ac5f91de89 | 127.28.1.129:39637 | HEALTHY | <none> | 2 | 0
57210dc0a07a4851a43be23a279eb836 | 127.28.1.131:39451 | HEALTHY | <none> | 3 | 0
6f37f9ed957342dba1271066c1bbeade | 127.28.1.134:41579 | HEALTHY | <none> | 1 | 0
9204a82012ac492dbc0da9c84c51dabf | 127.28.1.133:42523 | HEALTHY | <none> | 5 | 0
9fc0640dbc0d4208bfb536949d1bb0c4 | 127.28.1.132:41135 | HEALTHY | <none> | 2 | 0
fe99ab4507a24af4ba7fa5b5278c5ac6 | 127.28.1.135:36533 | HEALTHY | <none> | 0 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 7
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 7 server(s) checked
local_ip_for_outbound_sockets | 127.28.1.129 | experimental | 127.28.1.129:39637
local_ip_for_outbound_sockets | 127.28.1.130 | experimental | 127.28.1.130:46291
local_ip_for_outbound_sockets | 127.28.1.131 | experimental | 127.28.1.131:39451
local_ip_for_outbound_sockets | 127.28.1.132 | experimental | 127.28.1.132:41135
local_ip_for_outbound_sockets | 127.28.1.133 | experimental | 127.28.1.133:42523
local_ip_for_outbound_sockets | 127.28.1.134 | experimental | 127.28.1.134:41579
local_ip_for_outbound_sockets | 127.28.1.135 | experimental | 127.28.1.135:36533
never_fsync | true | unsafe,advanced | all 7 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 7 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 7 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 7 server(s) checked
server_dump_info_format | pb | hidden | all 7 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.28.1.129:39637
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.28.1.130:46291
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.28.1.131:39451
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.28.1.132:41135
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.28.1.133:42523
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.28.1.134:41579
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.28.1.135:36533
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:41967 | all 7 server(s) checked
time_source | builtin | all 7 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 8 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 11
Median | 11
Third Quartile | 12
Maximum | 12
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 7
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240501 08:12:02.431017 28678 log_verifier.cc:126] Checking tablet 120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:12:02.903864 28678 log_verifier.cc:177] Verified matching terms for 508 ops in tablet 120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:12:02.905285 28678 log_verifier.cc:126] Checking tablet 1280b2f88bbc470bbc9bc49c95a4b108
I20240501 08:12:03.267731 28678 log_verifier.cc:177] Verified matching terms for 372 ops in tablet 1280b2f88bbc470bbc9bc49c95a4b108
I20240501 08:12:03.268651 28678 log_verifier.cc:126] Checking tablet 1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:03.308817 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:03.308918 28678 log_verifier.cc:126] Checking tablet 3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:03.652781 28678 log_verifier.cc:177] Verified matching terms for 353 ops in tablet 3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:03.653650 28678 log_verifier.cc:126] Checking tablet 3c02021c881348748fa99b0df2dfb228
I20240501 08:12:04.027715 28678 log_verifier.cc:177] Verified matching terms for 407 ops in tablet 3c02021c881348748fa99b0df2dfb228
I20240501 08:12:04.028761 28678 log_verifier.cc:126] Checking tablet 568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:04.386034 28678 log_verifier.cc:177] Verified matching terms for 326 ops in tablet 568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:04.386837 28678 log_verifier.cc:126] Checking tablet 5730faa05092410da14e00e24ad74c86
I20240501 08:12:04.958374 28678 log_verifier.cc:177] Verified matching terms for 528 ops in tablet 5730faa05092410da14e00e24ad74c86
I20240501 08:12:04.959616 28678 log_verifier.cc:126] Checking tablet 6920f9b58f814b69a3cab12f5fa3de91
I20240501 08:12:05.356854 28678 log_verifier.cc:177] Verified matching terms for 375 ops in tablet 6920f9b58f814b69a3cab12f5fa3de91
I20240501 08:12:05.357764 28678 log_verifier.cc:126] Checking tablet 8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:05.772400 28678 log_verifier.cc:177] Verified matching terms for 444 ops in tablet 8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:05.773447 28678 log_verifier.cc:126] Checking tablet 9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:12:06.213737 28678 log_verifier.cc:177] Verified matching terms for 459 ops in tablet 9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:12:06.215363 28678 log_verifier.cc:126] Checking tablet 9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:06.632211 28678 log_verifier.cc:177] Verified matching terms for 443 ops in tablet 9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:06.633225 28678 log_verifier.cc:126] Checking tablet a465d27916734303bd9d1bed2a3dfb14
I20240501 08:12:06.998854 28678 log_verifier.cc:177] Verified matching terms for 390 ops in tablet a465d27916734303bd9d1bed2a3dfb14
I20240501 08:12:06.999784 28678 log_verifier.cc:126] Checking tablet b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:07.373332 28678 log_verifier.cc:177] Verified matching terms for 374 ops in tablet b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:07.374320 28678 log_verifier.cc:126] Checking tablet bc20191679084ecfbd360e23d68abd62
I20240501 08:12:07.754035 28678 log_verifier.cc:177] Verified matching terms for 379 ops in tablet bc20191679084ecfbd360e23d68abd62
I20240501 08:12:07.754897 28678 log_verifier.cc:126] Checking tablet e2393202bb014986b2496d3c99c65962
I20240501 08:12:08.077836 28678 log_verifier.cc:177] Verified matching terms for 354 ops in tablet e2393202bb014986b2496d3c99c65962
I20240501 08:12:08.078646 28678 log_verifier.cc:126] Checking tablet e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:08.520762 28678 log_verifier.cc:177] Verified matching terms for 499 ops in tablet e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:08.534119 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.136:0
--local_ip_for_outbound_sockets=127.28.1.136
--webserver_interface=127.28.1.136
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--builtin_ntp_servers=127.28.1.148:41967
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:12:08.893894 1755 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:08.894171 1755 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:08.894304 1755 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:08.902211 1755 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:08.902335 1755 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:08.902485 1755 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.136
I20240501 08:12:08.912343 1755 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:41967
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.28.1.136
--webserver_port=0
--tserver_master_addrs=127.28.1.190:35983
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.136
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:08.913652 1755 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:08.915336 1755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:08.927906 1760 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:08.928345 1763 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:08.928766 1761 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:08.929551 1755 server_base.cc:1034] running on GCE node
I20240501 08:12:08.930254 1755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:08.931233 1755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:08.932439 1755 hybrid_clock.cc:648] HybridClock initialized: now 1714551128932394 us; error 33 us; skew 500 ppm
I20240501 08:12:08.932745 1755 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:08.938647 1755 webserver.cc:460] Webserver started at http://127.28.1.136:43355/ using document root <none> and password file <none>
I20240501 08:12:08.939343 1755 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:08.939436 1755 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:08.939699 1755 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:08.943857 1755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "096aec539f1541b4b3049afbdcf36a81"
format_stamp: "Formatted at 2024-05-01 08:12:08 on dist-test-slave-jj4n"
I20240501 08:12:08.944641 1755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "096aec539f1541b4b3049afbdcf36a81"
format_stamp: "Formatted at 2024-05-01 08:12:08 on dist-test-slave-jj4n"
I20240501 08:12:08.949427 1755 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20240501 08:12:08.953022 1769 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:08.954384 1755 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240501 08:12:08.954526 1755 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "096aec539f1541b4b3049afbdcf36a81"
format_stamp: "Formatted at 2024-05-01 08:12:08 on dist-test-slave-jj4n"
I20240501 08:12:08.954728 1755 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:08.971727 1755 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:08.972558 1755 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:08.972829 1755 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:08.973788 1755 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:08.975215 1755 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:08.975323 1755 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:08.975440 1755 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:08.975517 1755 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:09.045903 1755 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.136:43857
I20240501 08:12:09.045969 1881 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.136:43857 every 8 connection(s)
I20240501 08:12:09.047628 1755 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240501 08:12:09.052160 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 1755
I20240501 08:12:09.052366 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240501 08:12:09.066808 1882 heartbeater.cc:347] Connected to a master server at 127.28.1.190:35983
I20240501 08:12:09.067104 1882 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:09.068006 1882 heartbeater.cc:511] Master 127.28.1.190:35983 requested a full tablet report, sending...
I20240501 08:12:09.069283 30559 ts_manager.cc:194] Registered new tserver with Master: 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857)
I20240501 08:12:09.070115 30559 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.136:35083
I20240501 08:12:09.568904 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:09.573037 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:09.585932 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:09.585995 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:09.628350 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:09.692503 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:09.694559 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:09.697878 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:09.861107 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:09.869976 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:09.874729 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:09.893339 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:09.898017 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:09.910987 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:09.926079 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:09.938555 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:10.034193 31224 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 353, Committed index: 353, Last appended: 1.353, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:10.037549 32298 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 353. Preceding OpId from leader: term: 1 index: 354. (index mismatch)
I20240501 08:12:10.037771 30965 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 353. Preceding OpId from leader: term: 1 index: 354. (index mismatch)
I20240501 08:12:10.038048 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 354, Last known committed idx: 353, Time since last communication: 0.000s
I20240501 08:12:10.038811 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 354, Last known committed idx: 353, Time since last communication: 0.000s
I20240501 08:12:10.044276 30707 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 353. Preceding OpId from leader: term: 1 index: 354. (index mismatch)
I20240501 08:12:10.044420 30838 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 353. Preceding OpId from leader: term: 1 index: 354. (index mismatch)
I20240501 08:12:10.048063 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 354, Last known committed idx: 353, Time since last communication: 0.000s
I20240501 08:12:10.048619 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 354, Last known committed idx: 353, Time since last communication: 0.000s
I20240501 08:12:10.056883 1980 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.354: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.057621 30707 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.354: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
W20240501 08:12:10.059243 31164 consensus_peers.cc:485] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3b8a23ca81da4039b7c18c84b2f2d8fb. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:10.063198 30560 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.063261 32298 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.354: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.073027 1882 heartbeater.cc:503] Master 127.28.1.190:35983 was elected leader, sending a full tablet report...
I20240501 08:12:10.073851 30965 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.354: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.077334 30840 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.354: config changed from index 126 to 354, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.096731 30707 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 374, Committed index: 374, Last appended: 1.374, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:10.100394 31100 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 1 index: 375. (index mismatch)
I20240501 08:12:10.100598 32298 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 1 index: 375. (index mismatch)
I20240501 08:12:10.100981 1948 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 374, Time since last communication: 0.000s
I20240501 08:12:10.101266 1674 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 374, Time since last communication: 0.000s
W20240501 08:12:10.107702 30643 consensus_peers.cc:485] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b3a5e3d4cc604fcd90a0efeea3946acd. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:10.108999 31224 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 1 index: 375. (index mismatch)
I20240501 08:12:10.109138 31360 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 1 index: 375. (index mismatch)
I20240501 08:12:10.109581 1997 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 374, Time since last communication: 0.000s
I20240501 08:12:10.109907 1674 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 374, Time since last communication: 0.000s
I20240501 08:12:10.112323 1948 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.375: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.112991 31360 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.375: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.113569 31224 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.375: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.113032 31100 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.375: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.119035 30563 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.119601 32298 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.375: config changed from index 198 to 375, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.143724 31100 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 443, Committed index: 443, Last appended: 2.443, Last appended by leader: 270, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:10.146842 31360 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 443. Preceding OpId from leader: term: 2 index: 444. (index mismatch)
I20240501 08:12:10.147667 30707 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 443. Preceding OpId from leader: term: 2 index: 444. (index mismatch)
I20240501 08:12:10.147905 32298 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 443. Preceding OpId from leader: term: 2 index: 444. (index mismatch)
I20240501 08:12:10.148592 1747 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 444, Last known committed idx: 443, Time since last communication: 0.000s
I20240501 08:12:10.148972 31224 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 443. Preceding OpId from leader: term: 2 index: 444. (index mismatch)
I20240501 08:12:10.149160 1747 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 444, Last known committed idx: 443, Time since last communication: 0.000s
I20240501 08:12:10.149601 1747 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 444, Last known committed idx: 443, Time since last communication: 0.000s
I20240501 08:12:10.150543 1747 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 444, Last known committed idx: 443, Time since last communication: 0.000s
W20240501 08:12:10.165311 31034 consensus_peers.cc:485] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9ff3c78386224499b7022dc9334ddd33. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:10.166862 2010 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 LEADER]: Committing config change with OpId 2.444: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.167820 30707 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Committing config change with OpId 2.444: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.168462 31224 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Committing config change with OpId 2.444: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.171334 31360 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Committing config change with OpId 2.444: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.172447 30560 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 2 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.175277 32298 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Committing config change with OpId 2.444: config changed from index 273 to 444, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.190387 31224 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 499, Committed index: 499, Last appended: 1.499, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:10.193449 31100 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
W20240501 08:12:10.193702 31164 consensus_peers.cc:485] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e61aa5646d3947c7b68bef19ee280c6e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:10.193929 32298 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240501 08:12:10.194003 2024 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240501 08:12:10.194442 1636 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240501 08:12:10.195389 30965 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240501 08:12:10.195482 30707 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240501 08:12:10.195936 2024 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240501 08:12:10.196471 2024 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240501 08:12:10.200384 1980 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.500: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.200959 32298 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.205664 30707 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.205834 31100 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.207844 30563 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:10.208678 30965 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index 250 to 500, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.229796 31224 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 379, Committed index: 379, Last appended: 1.379, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:10.232874 30840 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240501 08:12:10.232940 30707 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240501 08:12:10.233549 1980 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240501 08:12:10.233947 32298 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240501 08:12:10.234489 31360 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
W20240501 08:12:10.234664 31164 consensus_peers.cc:485] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bc20191679084ecfbd360e23d68abd62. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:10.234786 1636 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240501 08:12:10.235416 1636 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240501 08:12:10.235633 1980 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240501 08:12:10.253476 1952 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.380: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.254232 32298 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.254577 30840 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.254436 31360 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.259820 30560 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:10.259425 30707 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 140 to 380, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:10.530478 2077 ts_tablet_manager.cc:902] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637)
I20240501 08:12:10.532245 2077 tablet_copy_client.cc:308] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:39637
I20240501 08:12:10.539356 30729 tablet_copy_service.cc:140] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Received BeginTabletCopySession request for tablet b3a5e3d4cc604fcd90a0efeea3946acd from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:36429)
I20240501 08:12:10.539570 30729 tablet_copy_service.cc:161] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Beginning new tablet copy session on tablet b3a5e3d4cc604fcd90a0efeea3946acd from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:36429: session id = 096aec539f1541b4b3049afbdcf36a81-b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:10.542366 30729 tablet_copy_source_session.cc:205] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:10.547729 2077 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3a5e3d4cc604fcd90a0efeea3946acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:10.560212 2077 tablet_copy_client.cc:783] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:10.560671 2077 tablet_copy_client.cc:647] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:10.564334 2077 tablet_copy_client.cc:523] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:10.569195 2077 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:10.620000 2077 log.cc:824] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:10.621418 2097 ts_tablet_manager.cc:902] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135)
I20240501 08:12:10.623075 2097 tablet_copy_client.cc:308] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:41135
I20240501 08:12:10.630152 31120 tablet_copy_service.cc:140] P 9fc0640dbc0d4208bfb536949d1bb0c4: Received BeginTabletCopySession request for tablet 9ff3c78386224499b7022dc9334ddd33 from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:51647)
I20240501 08:12:10.630343 31120 tablet_copy_service.cc:161] P 9fc0640dbc0d4208bfb536949d1bb0c4: Beginning new tablet copy session on tablet 9ff3c78386224499b7022dc9334ddd33 from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:51647: session id = 096aec539f1541b4b3049afbdcf36a81-9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:10.632483 31120 tablet_copy_source_session.cc:205] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:10.633862 2097 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ff3c78386224499b7022dc9334ddd33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:10.638597 2097 tablet_copy_client.cc:783] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:10.638835 2097 tablet_copy_client.cc:647] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:10.641785 2097 tablet_copy_client.cc:523] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:10.644562 2097 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:10.660629 2102 ts_tablet_manager.cc:902] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:12:10.676321 2102 tablet_copy_client.cc:308] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:12:10.683873 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:33953)
I20240501 08:12:10.684096 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet 3b8a23ca81da4039b7c18c84b2f2d8fb from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:33953: session id = 096aec539f1541b4b3049afbdcf36a81-3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:10.686970 31250 tablet_copy_source_session.cc:205] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:10.688457 2102 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:10.695427 2102 tablet_copy_client.cc:783] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:10.695705 2102 tablet_copy_client.cc:647] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:10.700074 2102 tablet_copy_client.cc:523] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:10.704087 2102 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:10.845289 2114 ts_tablet_manager.cc:902] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:12:10.847790 2113 ts_tablet_manager.cc:902] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:12:10.877611 2113 tablet_copy_client.cc:308] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:12:10.878624 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet bc20191679084ecfbd360e23d68abd62 from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:33953)
I20240501 08:12:10.878796 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet bc20191679084ecfbd360e23d68abd62 from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:33953: session id = 096aec539f1541b4b3049afbdcf36a81-bc20191679084ecfbd360e23d68abd62
I20240501 08:12:10.882748 31250 tablet_copy_source_session.cc:205] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:10.884639 2113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc20191679084ecfbd360e23d68abd62. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:10.885301 2114 tablet_copy_client.cc:308] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:12:10.886073 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet e61aa5646d3947c7b68bef19ee280c6e from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:33953)
I20240501 08:12:10.886307 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet e61aa5646d3947c7b68bef19ee280c6e from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:33953: session id = 096aec539f1541b4b3049afbdcf36a81-e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:10.889374 31250 tablet_copy_source_session.cc:205] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:10.890722 2114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e61aa5646d3947c7b68bef19ee280c6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:10.891173 2113 tablet_copy_client.cc:783] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:10.891427 2113 tablet_copy_client.cc:647] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:10.894196 2113 tablet_copy_client.cc:523] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:10.897840 2113 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:10.902849 2114 tablet_copy_client.cc:783] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:10.904923 2114 tablet_copy_client.cc:647] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:10.909605 2114 tablet_copy_client.cc:523] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:10.913386 2114 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:11.295996 2077 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=375 overwritten=0 applied=375 ignored=0} inserts{seen=1044 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:11.296618 2077 tablet_bootstrap.cc:492] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:11.297188 2077 ts_tablet_manager.cc:1367] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.728s user 0.375s sys 0.069s
I20240501 08:12:11.300645 2102 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=354 overwritten=0 applied=354 ignored=0} inserts{seen=982 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:11.301119 2102 tablet_bootstrap.cc:492] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:11.301446 2102 ts_tablet_manager.cc:1367] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.597s user 0.412s sys 0.047s
I20240501 08:12:11.307802 2102 raft_consensus.cc:348] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.308452 2102 raft_consensus.cc:729] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:11.309069 2102 consensus_queue.cc:260] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 354, Last appended: 1.354, Last appended by leader: 354, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 354 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.309847 2077 raft_consensus.cc:348] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.310115 2077 raft_consensus.cc:729] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:11.310343 2077 consensus_queue.cc:260] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 375, Last appended: 1.375, Last appended by leader: 375, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 375 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.320955 2077 ts_tablet_manager.cc:1398] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.024s user 0.009s sys 0.001s
I20240501 08:12:11.321941 30729 tablet_copy_service.cc:342] P 3170be3b2bbb4b3bbd8a37ac5f91de89: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-b3a5e3d4cc604fcd90a0efeea3946acd received from {username='slave'} at 127.28.1.136:36429
I20240501 08:12:11.322063 2102 ts_tablet_manager.cc:1398] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.020s user 0.003s sys 0.007s
I20240501 08:12:11.322080 30729 tablet_copy_service.cc:433] P 3170be3b2bbb4b3bbd8a37ac5f91de89: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-b3a5e3d4cc604fcd90a0efeea3946acd on tablet b3a5e3d4cc604fcd90a0efeea3946acd with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:11.322731 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-3b8a23ca81da4039b7c18c84b2f2d8fb received from {username='slave'} at 127.28.1.136:33953
I20240501 08:12:11.322854 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-3b8a23ca81da4039b7c18c84b2f2d8fb on tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:11.363517 1837 raft_consensus.cc:1203] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.374->[1.375-1.375] Dedup: 1.375->[]
I20240501 08:12:11.406746 2097 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=444 overwritten=0 applied=444 ignored=0} inserts{seen=1125 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:11.417641 2097 tablet_bootstrap.cc:492] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:11.418061 2097 ts_tablet_manager.cc:1367] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.774s user 0.482s sys 0.044s
I20240501 08:12:11.418920 2097 raft_consensus.cc:348] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.420887 2097 raft_consensus.cc:729] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:11.421178 2097 consensus_queue.cc:260] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 444, Last appended: 2.444, Last appended by leader: 444, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 444 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.421825 2097 ts_tablet_manager.cc:1398] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240501 08:12:11.423288 31120 tablet_copy_service.cc:342] P 9fc0640dbc0d4208bfb536949d1bb0c4: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-9ff3c78386224499b7022dc9334ddd33 received from {username='slave'} at 127.28.1.136:51647
I20240501 08:12:11.423420 31120 tablet_copy_service.cc:433] P 9fc0640dbc0d4208bfb536949d1bb0c4: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-9ff3c78386224499b7022dc9334ddd33 on tablet 9ff3c78386224499b7022dc9334ddd33 with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:11.552836 1837 raft_consensus.cc:1203] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.353->[1.354-1.354] Dedup: 1.354->[]
I20240501 08:12:11.573405 2113 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=380 overwritten=0 applied=380 ignored=0} inserts{seen=1008 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:11.573822 2113 tablet_bootstrap.cc:492] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:11.574093 2113 ts_tablet_manager.cc:1367] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.676s user 0.383s sys 0.094s
I20240501 08:12:11.574743 2113 raft_consensus.cc:348] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.574986 2113 raft_consensus.cc:729] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:11.575170 2113 consensus_queue.cc:260] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 380, Last appended: 1.380, Last appended by leader: 380, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.575700 2113 ts_tablet_manager.cc:1398] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:11.576607 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-bc20191679084ecfbd360e23d68abd62 received from {username='slave'} at 127.28.1.136:33953
I20240501 08:12:11.576757 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-bc20191679084ecfbd360e23d68abd62 on tablet bc20191679084ecfbd360e23d68abd62 with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:11.617236 1837 raft_consensus.cc:1203] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.443->[2.444-2.444] Dedup: 2.444->[]
I20240501 08:12:11.727550 2114 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=500 overwritten=0 applied=500 ignored=0} inserts{seen=1169 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:11.728019 2114 tablet_bootstrap.cc:492] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:11.728324 2114 ts_tablet_manager.cc:1367] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.815s user 0.502s sys 0.041s
I20240501 08:12:11.728933 2114 raft_consensus.cc:348] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.729182 2114 raft_consensus.cc:729] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:11.729387 2114 consensus_queue.cc:260] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 500, Last appended: 1.500, Last appended by leader: 500, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:11.729933 2114 ts_tablet_manager.cc:1398] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:11.730595 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-e61aa5646d3947c7b68bef19ee280c6e received from {username='slave'} at 127.28.1.136:33953
I20240501 08:12:11.730728 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-e61aa5646d3947c7b68bef19ee280c6e on tablet e61aa5646d3947c7b68bef19ee280c6e with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:11.833196 1837 raft_consensus.cc:1203] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.379->[1.380-1.380] Dedup: 1.380->[]
I20240501 08:12:11.838181 1837 raft_consensus.cc:1203] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.499->[1.500-1.500] Dedup: 1.500->[]
I20240501 08:12:11.854070 2204 raft_consensus.cc:1050] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:11.854889 2204 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 375, Committed index: 375, Last appended: 1.375, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:11.857610 32298 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 376. (index mismatch)
I20240501 08:12:11.857693 31100 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 376. (index mismatch)
I20240501 08:12:11.858045 31227 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 376. (index mismatch)
I20240501 08:12:11.858187 31360 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 376. (index mismatch)
I20240501 08:12:11.858233 2206 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240501 08:12:11.858546 2140 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240501 08:12:11.858812 2003 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240501 08:12:11.858964 1837 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 376. (index mismatch)
I20240501 08:12:11.859138 2204 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240501 08:12:11.859848 2207 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240501 08:12:11.864517 2206 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.868080 31227 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.868526 32298 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.869699 31360 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.872484 30566 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:11.874483 1836 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.878307 31100 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.376: config changed from index 375 to 376, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.890985 30707 consensus_queue.cc:237] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 376, Committed index: 376, Last appended: 1.376, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:11.893036 31100 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 376. Preceding OpId from leader: term: 1 index: 377. (index mismatch)
I20240501 08:12:11.893113 1836 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 376. Preceding OpId from leader: term: 1 index: 377. (index mismatch)
I20240501 08:12:11.893136 32298 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 376. Preceding OpId from leader: term: 1 index: 377. (index mismatch)
I20240501 08:12:11.893101 31227 raft_consensus.cc:1261] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 3170be3b2bbb4b3bbd8a37ac5f91de89: Log matching property violated. Preceding OpId in replica: term: 1 index: 376. Preceding OpId from leader: term: 1 index: 377. (index mismatch)
I20240501 08:12:11.893643 2140 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 377, Last known committed idx: 376, Time since last communication: 0.000s
I20240501 08:12:11.893906 2204 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 377, Last known committed idx: 376, Time since last communication: 0.000s
I20240501 08:12:11.894136 2003 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 377, Last known committed idx: 376, Time since last communication: 0.000s
I20240501 08:12:11.894389 2206 consensus_queue.cc:1035] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 377, Last known committed idx: 376, Time since last communication: 0.000s
I20240501 08:12:11.896946 2003 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 LEADER]: Committing config change with OpId 1.377: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.897651 31100 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.377: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.897764 1836 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.377: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.897701 32298 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.377: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.900837 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b3a5e3d4cc604fcd90a0efeea3946acd with cas_config_opid_index 376: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:11.903775 30559 catalog_manager.cc:5484] T b3a5e3d4cc604fcd90a0efeea3946acd P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New cstate: current_term: 1 leader_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" committed_config { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.905946 31227 raft_consensus.cc:2932] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.377: config changed from index 376 to 377, VOTER 6f37f9ed957342dba1271066c1bbeade (127.28.1.134) evicted. New config: { opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:11.907440 31338 tablet_service.cc:1514] Processing DeleteTablet for tablet b3a5e3d4cc604fcd90a0efeea3946acd with delete_type TABLET_DATA_TOMBSTONED (TS 6f37f9ed957342dba1271066c1bbeade not found in new config with opid_index 377) from {username='slave'} at 127.0.0.1:43272
I20240501 08:12:11.911854 2218 tablet_replica.cc:330] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: stopping tablet replica
I20240501 08:12:11.912057 2218 raft_consensus.cc:2229] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:11.912353 2218 raft_consensus.cc:2258] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:11.913771 2218 ts_tablet_manager.cc:1875] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:11.918439 2218 ts_tablet_manager.cc:1888] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.376
I20240501 08:12:11.918560 2218 log.cc:1196] T b3a5e3d4cc604fcd90a0efeea3946acd P 6f37f9ed957342dba1271066c1bbeade: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/wals/b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:11.919199 30554 catalog_manager.cc:4830] TS 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): tablet b3a5e3d4cc604fcd90a0efeea3946acd (table rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]) successfully deleted
I20240501 08:12:11.976281 1952 raft_consensus.cc:1050] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:11.977279 1952 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 354, Committed index: 354, Last appended: 1.354, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:11.980228 1836 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 354. Preceding OpId from leader: term: 1 index: 355. (index mismatch)
I20240501 08:12:11.980336 30965 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 354. Preceding OpId from leader: term: 1 index: 355. (index mismatch)
I20240501 08:12:11.980896 2203 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 355, Last known committed idx: 354, Time since last communication: 0.000s
I20240501 08:12:11.981117 32298 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 354. Preceding OpId from leader: term: 1 index: 355. (index mismatch)
I20240501 08:12:11.981642 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 355, Last known committed idx: 354, Time since last communication: 0.000s
I20240501 08:12:11.982118 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 355, Last known committed idx: 354, Time since last communication: 0.000s
I20240501 08:12:11.992089 30840 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 354. Preceding OpId from leader: term: 1 index: 355. (index mismatch)
I20240501 08:12:11.992197 30707 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 354. Preceding OpId from leader: term: 1 index: 355. (index mismatch)
I20240501 08:12:11.992868 2203 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 355, Last known committed idx: 354, Time since last communication: 0.000s
I20240501 08:12:11.994721 1636 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 355, Last known committed idx: 354, Time since last communication: 0.000s
I20240501 08:12:11.996340 1636 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.000900 1837 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.009289 32298 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.009791 30560 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:12.012075 30965 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.022245 30840 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.033393 31227 consensus_queue.cc:237] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 355, Committed index: 355, Last appended: 1.355, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.035563 32298 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 355. Preceding OpId from leader: term: 1 index: 356. (index mismatch)
I20240501 08:12:12.036146 1952 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 356, Last known committed idx: 355, Time since last communication: 0.000s
I20240501 08:12:12.040464 1837 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 355. Preceding OpId from leader: term: 1 index: 356. (index mismatch)
I20240501 08:12:12.040298 30707 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.355: config changed from index 354 to 355, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 355 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.042028 2225 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 356, Last known committed idx: 355, Time since last communication: 0.000s
I20240501 08:12:12.042982 30965 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 355. Preceding OpId from leader: term: 1 index: 356. (index mismatch)
I20240501 08:12:12.043550 1952 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 356, Last known committed idx: 355, Time since last communication: 0.000s
I20240501 08:12:12.045027 2203 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.356: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.049450 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with cas_config_opid_index 355: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:12.050107 30709 raft_consensus.cc:1261] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 355. Preceding OpId from leader: term: 1 index: 356. (index mismatch)
I20240501 08:12:12.050987 32298 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.356: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.053308 30560 catalog_manager.cc:5484] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:12.053181 1837 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.356: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.056640 2225 consensus_queue.cc:1035] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 356, Last known committed idx: 355, Time since last communication: 0.000s
I20240501 08:12:12.058657 30709 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.356: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.060145 30818 tablet_service.cc:1514] Processing DeleteTablet for tablet 3b8a23ca81da4039b7c18c84b2f2d8fb with delete_type TABLET_DATA_TOMBSTONED (TS 1a388864321846929919a37b570e8138 not found in new config with opid_index 356) from {username='slave'} at 127.0.0.1:37872
I20240501 08:12:12.059919 30969 raft_consensus.cc:2932] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.356: config changed from index 355 to 356, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 356 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.077018 2240 tablet_replica.cc:330] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: stopping tablet replica
I20240501 08:12:12.077463 2240 raft_consensus.cc:2229] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:12.077782 2240 raft_consensus.cc:2258] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:12.080235 2240 ts_tablet_manager.cc:1875] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:12.086712 2240 ts_tablet_manager.cc:1888] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.355
I20240501 08:12:12.086810 2240 log.cc:1196] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 1a388864321846929919a37b570e8138: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:12.087489 30555 catalog_manager.cc:4830] TS 1a388864321846929919a37b570e8138 (127.28.1.130:46291): tablet 3b8a23ca81da4039b7c18c84b2f2d8fb (table rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]) successfully deleted
I20240501 08:12:12.118606 31227 consensus_queue.cc:237] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.121088 32298 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:12.121279 30840 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:12.121335 30709 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240501 08:12:12.121502 31164 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1469837fa8424b94b9ec4bbe57fdf740. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:12.121609 30969 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:12.121991 2203 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240501 08:12:12.122262 1952 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240501 08:12:12.122773 1636 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240501 08:12:12.123044 2225 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240501 08:12:12.140092 1636 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.140780 32298 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.145069 30709 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.148567 30559 catalog_manager.cc:5484] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.148300 30969 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.152436 30840 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.171530 2264 raft_consensus.cc:1050] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:12.172601 2264 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 444, Committed index: 444, Last appended: 2.444, Last appended by leader: 270, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.175506 1837 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 LEARNER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 444. Preceding OpId from leader: term: 2 index: 445. (index mismatch)
I20240501 08:12:12.176559 30709 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 444. Preceding OpId from leader: term: 2 index: 445. (index mismatch)
I20240501 08:12:12.176965 31227 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 444. Preceding OpId from leader: term: 2 index: 445. (index mismatch)
I20240501 08:12:12.177119 31360 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 444. Preceding OpId from leader: term: 2 index: 445. (index mismatch)
I20240501 08:12:12.177572 2266 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.178099 32298 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 444. Preceding OpId from leader: term: 2 index: 445. (index mismatch)
I20240501 08:12:12.178117 2177 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.178795 2177 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.179754 2266 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.182242 2177 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.212360 2266 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 LEADER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.213439 31360 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.213856 30709 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.217191 1837 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.217157 32298 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.219179 30559 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.224618 31227 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Committing config change with OpId 2.445: config changed from index 444 to 445, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.229717 31100 consensus_queue.cc:237] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 445, Committed index: 445, Last appended: 2.445, Last appended by leader: 270, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.232434 31353 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 445. Preceding OpId from leader: term: 2 index: 446. (index mismatch)
I20240501 08:12:12.233027 1837 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 445. Preceding OpId from leader: term: 2 index: 446. (index mismatch)
I20240501 08:12:12.233971 2265 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:12.234222 30709 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 445. Preceding OpId from leader: term: 2 index: 446. (index mismatch)
I20240501 08:12:12.234665 2265 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:12.235744 2267 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:12.237125 32298 raft_consensus.cc:1261] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Refusing update from remote peer 9fc0640dbc0d4208bfb536949d1bb0c4: Log matching property violated. Preceding OpId in replica: term: 2 index: 445. Preceding OpId from leader: term: 2 index: 446. (index mismatch)
I20240501 08:12:12.237882 2267 consensus_queue.cc:1035] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:12.240785 30969 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 444, Committed index: 444, Last appended: 1.444, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.243811 2267 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 LEADER]: Committing config change with OpId 2.446: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.247005 32293 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Committing config change with OpId 2.446: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.247169 31353 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Committing config change with OpId 2.446: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.251627 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9ff3c78386224499b7022dc9334ddd33 with cas_config_opid_index 445: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:12.252156 30705 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 2 FOLLOWER]: Committing config change with OpId 2.446: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.255491 1836 raft_consensus.cc:2932] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Committing config change with OpId 2.446: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.256592 30560 catalog_manager.cc:5484] T 9ff3c78386224499b7022dc9334ddd33 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 445 to 446, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New cstate: current_term: 2 leader_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" committed_config { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.261355 31209 tablet_service.cc:1514] Processing DeleteTablet for tablet 9ff3c78386224499b7022dc9334ddd33 with delete_type TABLET_DATA_TOMBSTONED (TS 9204a82012ac492dbc0da9c84c51dabf not found in new config with opid_index 446) from {username='slave'} at 127.0.0.1:57760
I20240501 08:12:12.261607 31227 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 444. Preceding OpId from leader: term: 1 index: 445. (index mismatch)
I20240501 08:12:12.262452 32293 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 444. Preceding OpId from leader: term: 1 index: 445. (index mismatch)
I20240501 08:12:12.263890 30840 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 444. Preceding OpId from leader: term: 1 index: 445. (index mismatch)
I20240501 08:12:12.263787 2108 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.264472 2108 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.265429 1952 raft_consensus.cc:1050] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:12.266427 1952 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 380, Committed index: 380, Last appended: 1.380, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.276002 31100 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 444. Preceding OpId from leader: term: 1 index: 445. (index mismatch)
I20240501 08:12:12.276661 2108 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.277263 2281 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 445, Last known committed idx: 444, Time since last communication: 0.000s
I20240501 08:12:12.278728 1836 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 380. Preceding OpId from leader: term: 1 index: 381. (index mismatch)
I20240501 08:12:12.279374 2225 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 381, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.279716 2108 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.445: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.281311 31353 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 380. Preceding OpId from leader: term: 1 index: 381. (index mismatch)
I20240501 08:12:12.281904 2225 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 381, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.285197 32293 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 380. Preceding OpId from leader: term: 1 index: 381. (index mismatch)
I20240501 08:12:12.285694 2225 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 381, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.285470 30840 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.445: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.289655 30705 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 380. Preceding OpId from leader: term: 1 index: 381. (index mismatch)
I20240501 08:12:12.290314 2225 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 381, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.293159 30559 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.301084 32298 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.445: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.303522 30840 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 380. Preceding OpId from leader: term: 1 index: 381. (index mismatch)
I20240501 08:12:12.304212 1952 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 381, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.318040 2282 tablet_replica.cc:330] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: stopping tablet replica
I20240501 08:12:12.318354 2282 raft_consensus.cc:2229] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:12.318732 2282 raft_consensus.cc:2258] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:12.318647 2225 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.321063 31227 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.445: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.321867 31100 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.445: config changed from index 160 to 445, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.322113 1837 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
W20240501 08:12:12.327483 30904 consensus_peers.cc:485] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8302dd0a0d2e4a20bdc00e27da939e8a. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:12.327778 30561 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 reported cstate change: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.331746 2282 ts_tablet_manager.cc:1875] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:12.331665 31360 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.338927 30840 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.341140 2282 ts_tablet_manager.cc:1888] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.445
I20240501 08:12:12.341367 2282 log.cc:1196] T 9ff3c78386224499b7022dc9334ddd33 P 9204a82012ac492dbc0da9c84c51dabf: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:12.342306 31224 consensus_queue.cc:237] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 381, Committed index: 381, Last appended: 1.381, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.344290 30553 catalog_manager.cc:4830] TS 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): tablet 9ff3c78386224499b7022dc9334ddd33 (table rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]) successfully deleted
I20240501 08:12:12.346273 32298 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 381. Preceding OpId from leader: term: 1 index: 382. (index mismatch)
I20240501 08:12:12.347084 31360 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 381. Preceding OpId from leader: term: 1 index: 382. (index mismatch)
I20240501 08:12:12.347339 1952 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 382, Last known committed idx: 380, Time since last communication: 0.000s
I20240501 08:12:12.347821 1952 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 382, Last known committed idx: 381, Time since last communication: 0.000s
I20240501 08:12:12.347914 1837 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 381. Preceding OpId from leader: term: 1 index: 382. (index mismatch)
I20240501 08:12:12.348834 2225 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 382, Last known committed idx: 381, Time since last communication: 0.000s
I20240501 08:12:12.349148 32298 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.351418 30705 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.381: config changed from index 380 to 381, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 381 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { replace: true } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.354135 30838 raft_consensus.cc:1261] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 381. Preceding OpId from leader: term: 1 index: 382. (index mismatch)
I20240501 08:12:12.357556 1952 consensus_queue.cc:1035] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 382, Last known committed idx: 381, Time since last communication: 0.000s
I20240501 08:12:12.361795 2225 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.382: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.363126 32293 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.382: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.365609 30838 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.382: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.367632 1837 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.382: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.370151 30563 catalog_manager.cc:5484] T bc20191679084ecfbd360e23d68abd62 P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.370776 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bc20191679084ecfbd360e23d68abd62 with cas_config_opid_index 381: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:12.376327 31360 raft_consensus.cc:2932] T bc20191679084ecfbd360e23d68abd62 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.382: config changed from index 381 to 382, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 382 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.381757 30687 tablet_service.cc:1514] Processing DeleteTablet for tablet bc20191679084ecfbd360e23d68abd62 with delete_type TABLET_DATA_TOMBSTONED (TS 3170be3b2bbb4b3bbd8a37ac5f91de89 not found in new config with opid_index 382) from {username='slave'} at 127.0.0.1:43428
I20240501 08:12:12.385813 1636 raft_consensus.cc:1050] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:12.386816 1636 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 500, Committed index: 500, Last appended: 1.500, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.397526 32293 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240501 08:12:12.397686 1837 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240501 08:12:12.398259 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.398550 30705 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240501 08:12:12.399384 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.399883 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.405727 31100 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240501 08:12:12.406306 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.414132 30969 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240501 08:12:12.416679 1636 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.421521 1952 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.424714 32298 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.431113 1837 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.432365 30563 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:12.438848 30705 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.450409 31100 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.451915 2305 tablet_replica.cc:330] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: stopping tablet replica
I20240501 08:12:12.452235 2305 raft_consensus.cc:2229] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:12.452550 2305 raft_consensus.cc:2258] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:12.454535 2305 ts_tablet_manager.cc:1875] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:12.459460 31227 consensus_queue.cc:237] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 501, Committed index: 501, Last appended: 1.501, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:12.461477 2305 ts_tablet_manager.cc:1888] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.381
I20240501 08:12:12.461704 2305 log.cc:1196] T bc20191679084ecfbd360e23d68abd62 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/bc20191679084ecfbd360e23d68abd62
I20240501 08:12:12.462436 1837 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 1 index: 502. (index mismatch)
I20240501 08:12:12.462800 30552 catalog_manager.cc:4830] TS 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): tablet bc20191679084ecfbd360e23d68abd62 (table rebalance_test_table_0 [id=d57de712d6664d73a00cf72cfbe56003]) successfully deleted
I20240501 08:12:12.463387 30705 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 1 index: 502. (index mismatch)
I20240501 08:12:12.463660 32293 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 1 index: 502. (index mismatch)
I20240501 08:12:12.463752 30965 raft_consensus.cc:1261] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 1 index: 502. (index mismatch)
I20240501 08:12:12.463915 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240501 08:12:12.464486 2203 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240501 08:12:12.464285 30969 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } attrs { replace: true } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.467232 2203 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 500, Time since last communication: 0.000s
I20240501 08:12:12.471297 2203 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.502: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.472041 1952 consensus_queue.cc:1035] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240501 08:12:12.474390 30705 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.502: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.474444 32293 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.502: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.477931 30965 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.502: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.480468 30560 catalog_manager.cc:5484] T e61aa5646d3947c7b68bef19ee280c6e P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.484391 31079 tablet_service.cc:1514] Processing DeleteTablet for tablet e61aa5646d3947c7b68bef19ee280c6e with delete_type TABLET_DATA_TOMBSTONED (TS 9fc0640dbc0d4208bfb536949d1bb0c4 not found in new config with opid_index 502) from {username='slave'} at 127.0.0.1:43748
I20240501 08:12:12.486774 2315 tablet_replica.cc:330] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: stopping tablet replica
I20240501 08:12:12.487049 2315 raft_consensus.cc:2229] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:12.487347 2315 raft_consensus.cc:2258] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:12.488868 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e61aa5646d3947c7b68bef19ee280c6e with cas_config_opid_index 501: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:12.490052 2315 ts_tablet_manager.cc:1875] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:12.491369 1837 raft_consensus.cc:2932] T e61aa5646d3947c7b68bef19ee280c6e P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.502: config changed from index 501 to 502, VOTER 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132) evicted. New config: { opid_index: 502 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:12.498473 2315 ts_tablet_manager.cc:1888] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.501
I20240501 08:12:12.498682 2315 log.cc:1196] T e61aa5646d3947c7b68bef19ee280c6e P 9fc0640dbc0d4208bfb536949d1bb0c4: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:12.499603 30553 catalog_manager.cc:4830] TS 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): tablet e61aa5646d3947c7b68bef19ee280c6e (table rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]) successfully deleted
I20240501 08:12:12.561007 31227 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 528, Committed index: 528, Last appended: 1.528, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.564179 31360 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 528. Preceding OpId from leader: term: 1 index: 529. (index mismatch)
I20240501 08:12:12.564399 30705 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 528. Preceding OpId from leader: term: 1 index: 529. (index mismatch)
I20240501 08:12:12.564651 30838 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 528. Preceding OpId from leader: term: 1 index: 529. (index mismatch)
I20240501 08:12:12.564692 2203 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 529, Last known committed idx: 528, Time since last communication: 0.000s
I20240501 08:12:12.565045 2258 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 529, Last known committed idx: 528, Time since last communication: 0.000s
I20240501 08:12:12.565395 1636 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 529, Last known committed idx: 528, Time since last communication: 0.000s
I20240501 08:12:12.566357 31100 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 528. Preceding OpId from leader: term: 1 index: 529. (index mismatch)
W20240501 08:12:12.567163 31164 consensus_peers.cc:485] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5730faa05092410da14e00e24ad74c86. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:12.567451 2258 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 529, Last known committed idx: 528, Time since last communication: 0.000s
I20240501 08:12:12.570024 1636 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.529: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.570636 30705 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.529: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.573408 31100 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.529: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.573633 30838 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.529: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.574805 30560 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:12.585780 31360 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.529: config changed from index 186 to 529, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.596221 30838 consensus_queue.cc:237] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 407, Committed index: 407, Last appended: 1.407, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.598940 30965 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 407. Preceding OpId from leader: term: 1 index: 408. (index mismatch)
I20240501 08:12:12.599377 31360 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 407. Preceding OpId from leader: term: 1 index: 408. (index mismatch)
I20240501 08:12:12.599793 30705 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 407. Preceding OpId from leader: term: 1 index: 408. (index mismatch)
I20240501 08:12:12.600246 2279 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 408, Last known committed idx: 407, Time since last communication: 0.000s
I20240501 08:12:12.600623 2348 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 408, Last known committed idx: 407, Time since last communication: 0.000s
I20240501 08:12:12.600950 1887 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 408, Last known committed idx: 407, Time since last communication: 0.000s
I20240501 08:12:12.602077 31227 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 407. Preceding OpId from leader: term: 1 index: 408. (index mismatch)
I20240501 08:12:12.602711 2348 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 408, Last known committed idx: 407, Time since last communication: 0.000s
I20240501 08:12:12.606058 2348 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.408: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.606602 31360 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.408: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
W20240501 08:12:12.609222 30774 consensus_peers.cc:485] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3c02021c881348748fa99b0df2dfb228. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:12.612718 30965 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.408: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.613765 30561 catalog_manager.cc:5484] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:12.612179 31227 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.408: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.620503 30705 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.408: config changed from index -1 to 408, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:12.680476 2374 ts_tablet_manager.cc:902] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:12:12.681636 2374 tablet_copy_client.cc:308] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:12:12.682214 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet 1469837fa8424b94b9ec4bbe57fdf740 from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:33953)
I20240501 08:12:12.682339 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet 1469837fa8424b94b9ec4bbe57fdf740 from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:33953: session id = 096aec539f1541b4b3049afbdcf36a81-1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:12.684194 31250 tablet_copy_source_session.cc:205] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:12.685242 2374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1469837fa8424b94b9ec4bbe57fdf740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:12.689798 2374 tablet_copy_client.cc:783] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:12.690006 2374 tablet_copy_client.cc:647] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:12.691397 2374 tablet_copy_client.cc:523] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:12.693570 2374 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:12.698127 2374 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:12.698432 2374 tablet_bootstrap.cc:492] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:12.698601 2374 ts_tablet_manager.cc:1367] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240501 08:12:12.699164 2374 raft_consensus.cc:348] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.699415 2374 raft_consensus.cc:729] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:12.699622 2374 consensus_queue.cc:260] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:12.700204 2374 ts_tablet_manager.cc:1398] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:12:12.700835 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-1469837fa8424b94b9ec4bbe57fdf740 received from {username='slave'} at 127.28.1.136:33953
I20240501 08:12:12.700924 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-1469837fa8424b94b9ec4bbe57fdf740 on tablet 1469837fa8424b94b9ec4bbe57fdf740 with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:12.706276 2374 ts_tablet_manager.cc:902] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:12:12.707095 2374 tablet_copy_client.cc:308] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:39451
I20240501 08:12:12.712203 30990 tablet_copy_service.cc:140] P 57210dc0a07a4851a43be23a279eb836: Received BeginTabletCopySession request for tablet 8302dd0a0d2e4a20bdc00e27da939e8a from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:40929)
I20240501 08:12:12.712343 30990 tablet_copy_service.cc:161] P 57210dc0a07a4851a43be23a279eb836: Beginning new tablet copy session on tablet 8302dd0a0d2e4a20bdc00e27da939e8a from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:40929: session id = 096aec539f1541b4b3049afbdcf36a81-8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:12.714532 30990 tablet_copy_source_session.cc:205] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:12.715581 2374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8302dd0a0d2e4a20bdc00e27da939e8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:12.719627 2374 tablet_copy_client.cc:783] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:12.719846 2374 tablet_copy_client.cc:647] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:12.722149 2374 tablet_copy_client.cc:523] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:12.724381 2374 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:12.971333 2406 ts_tablet_manager.cc:902] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
I20240501 08:12:12.977735 2406 tablet_copy_client.cc:308] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42523
I20240501 08:12:12.978394 31250 tablet_copy_service.cc:140] P 9204a82012ac492dbc0da9c84c51dabf: Received BeginTabletCopySession request for tablet 5730faa05092410da14e00e24ad74c86 from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:33953)
I20240501 08:12:12.978534 31250 tablet_copy_service.cc:161] P 9204a82012ac492dbc0da9c84c51dabf: Beginning new tablet copy session on tablet 5730faa05092410da14e00e24ad74c86 from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:33953: session id = 096aec539f1541b4b3049afbdcf36a81-5730faa05092410da14e00e24ad74c86
I20240501 08:12:12.980410 31250 tablet_copy_source_session.cc:205] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:12.981531 2406 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5730faa05092410da14e00e24ad74c86. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:12.985977 2406 tablet_copy_client.cc:783] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:12.986189 2406 tablet_copy_client.cc:647] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:12.989173 2406 tablet_copy_client.cc:523] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:12.991495 2406 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:13.125123 2374 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=445 overwritten=0 applied=445 ignored=0} inserts{seen=1114 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:13.125573 2374 tablet_bootstrap.cc:492] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:13.125914 2374 ts_tablet_manager.cc:1367] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.402s user 0.370s sys 0.027s
I20240501 08:12:13.126505 2374 raft_consensus.cc:348] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.126746 2374 raft_consensus.cc:729] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:13.126930 2374 consensus_queue.cc:260] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 445, Last appended: 1.445, Last appended by leader: 445, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 445 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.127393 2374 ts_tablet_manager.cc:1398] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:13.128026 30990 tablet_copy_service.cc:342] P 57210dc0a07a4851a43be23a279eb836: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-8302dd0a0d2e4a20bdc00e27da939e8a received from {username='slave'} at 127.28.1.136:40929
I20240501 08:12:13.128176 30990 tablet_copy_service.cc:433] P 57210dc0a07a4851a43be23a279eb836: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-8302dd0a0d2e4a20bdc00e27da939e8a on tablet 8302dd0a0d2e4a20bdc00e27da939e8a with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:13.156690 2374 ts_tablet_manager.cc:902] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:12:13.169170 2374 tablet_copy_client.cc:308] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:46291
I20240501 08:12:13.174175 1837 raft_consensus.cc:1203] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240501 08:12:13.176625 30860 tablet_copy_service.cc:140] P 1a388864321846929919a37b570e8138: Received BeginTabletCopySession request for tablet 3c02021c881348748fa99b0df2dfb228 from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:52233)
I20240501 08:12:13.176782 30860 tablet_copy_service.cc:161] P 1a388864321846929919a37b570e8138: Beginning new tablet copy session on tablet 3c02021c881348748fa99b0df2dfb228 from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:52233: session id = 096aec539f1541b4b3049afbdcf36a81-3c02021c881348748fa99b0df2dfb228
I20240501 08:12:13.179466 30860 tablet_copy_source_session.cc:205] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:13.181602 2374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c02021c881348748fa99b0df2dfb228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:13.187565 2374 tablet_copy_client.cc:783] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:13.187817 2374 tablet_copy_client.cc:647] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:13.191509 2374 tablet_copy_client.cc:523] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:13.194801 2374 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:13.248286 1837 raft_consensus.cc:1203] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.444->[1.445-1.445] Dedup: 1.445->[]
I20240501 08:12:13.551750 2406 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=529 overwritten=0 applied=529 ignored=0} inserts{seen=1157 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:13.552181 2406 tablet_bootstrap.cc:492] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:13.552477 2406 ts_tablet_manager.cc:1367] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.561s user 0.518s sys 0.028s
I20240501 08:12:13.553153 2406 raft_consensus.cc:348] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.553401 2406 raft_consensus.cc:729] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:13.553618 2406 consensus_queue.cc:260] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 529, Last appended: 1.529, Last appended by leader: 529, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 529 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.554212 2406 ts_tablet_manager.cc:1398] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:13.554906 31250 tablet_copy_service.cc:342] P 9204a82012ac492dbc0da9c84c51dabf: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-5730faa05092410da14e00e24ad74c86 received from {username='slave'} at 127.28.1.136:33953
I20240501 08:12:13.555023 31250 tablet_copy_service.cc:433] P 9204a82012ac492dbc0da9c84c51dabf: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-5730faa05092410da14e00e24ad74c86 on tablet 5730faa05092410da14e00e24ad74c86 with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:13.593050 1837 raft_consensus.cc:1203] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.528->[1.529-1.529] Dedup: 1.529->[]
I20240501 08:12:13.597892 2374 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=408 overwritten=0 applied=408 ignored=0} inserts{seen=1039 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:13.598278 2374 tablet_bootstrap.cc:492] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:13.598524 2374 ts_tablet_manager.cc:1367] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.404s user 0.393s sys 0.008s
I20240501 08:12:13.599120 2374 raft_consensus.cc:348] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.599318 2374 raft_consensus.cc:729] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:13.599503 2374 consensus_queue.cc:260] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 408, Last appended: 1.408, Last appended by leader: 408, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 408 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:13.600013 2374 ts_tablet_manager.cc:1398] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240501 08:12:13.600791 30860 tablet_copy_service.cc:342] P 1a388864321846929919a37b570e8138: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-3c02021c881348748fa99b0df2dfb228 received from {username='slave'} at 127.28.1.136:52233
I20240501 08:12:13.600927 30860 tablet_copy_service.cc:433] P 1a388864321846929919a37b570e8138: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-3c02021c881348748fa99b0df2dfb228 on tablet 3c02021c881348748fa99b0df2dfb228 with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:13.609273 1636 raft_consensus.cc:1050] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:13.609992 1636 consensus_queue.cc:237] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:13.612326 1837 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240501 08:12:13.612845 1952 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240501 08:12:13.612991 32293 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240501 08:12:13.613538 30838 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240501 08:12:13.613993 30965 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240501 08:12:13.614022 30705 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240501 08:12:13.614481 2464 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240501 08:12:13.614698 1952 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240501 08:12:13.614943 1636 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240501 08:12:13.622887 2463 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240501 08:12:13.625945 2463 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.626402 32293 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.626361 1837 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.631373 30563 catalog_manager.cc:5484] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 reported cstate change: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.635015 30965 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.635269 30838 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.635560 30705 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.646227 31227 consensus_queue.cc:237] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:13.648226 30838 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:13.648419 30705 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:13.648715 1636 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:13.648895 32293 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:13.649479 1636 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:13.649863 1837 raft_consensus.cc:1261] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:13.649825 2463 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:13.650516 2463 consensus_queue.cc:1035] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:13.651791 1636 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.652416 32293 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.654686 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1469837fa8424b94b9ec4bbe57fdf740 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:13.655334 30705 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.656288 30559 catalog_manager.cc:5484] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:13.657421 1837 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.661214 30948 tablet_service.cc:1514] Processing DeleteTablet for tablet 1469837fa8424b94b9ec4bbe57fdf740 with delete_type TABLET_DATA_TOMBSTONED (TS 57210dc0a07a4851a43be23a279eb836 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:44190
I20240501 08:12:13.662138 30838 raft_consensus.cc:2932] T 1469837fa8424b94b9ec4bbe57fdf740 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 57210dc0a07a4851a43be23a279eb836 (127.28.1.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.671303 2475 tablet_replica.cc:330] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: stopping tablet replica
I20240501 08:12:13.671504 2475 raft_consensus.cc:2229] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:13.671756 2475 raft_consensus.cc:2258] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:13.673233 2475 ts_tablet_manager.cc:1875] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:13.677239 2475 ts_tablet_manager.cc:1888] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240501 08:12:13.677333 2475 log.cc:1196] T 1469837fa8424b94b9ec4bbe57fdf740 P 57210dc0a07a4851a43be23a279eb836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:13.677887 30552 catalog_manager.cc:4830] TS 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): tablet 1469837fa8424b94b9ec4bbe57fdf740 (table TestTable [id=c0be68555ded4b888585e5508b80e2db]) successfully deleted
I20240501 08:12:13.747387 2437 raft_consensus.cc:1050] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:13.748445 2437 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 445, Committed index: 445, Last appended: 1.445, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:13.751493 31227 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 445. Preceding OpId from leader: term: 1 index: 446. (index mismatch)
I20240501 08:12:13.756300 2494 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:13.757301 32293 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 445. Preceding OpId from leader: term: 1 index: 446. (index mismatch)
I20240501 08:12:13.757553 1837 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 445. Preceding OpId from leader: term: 1 index: 446. (index mismatch)
I20240501 08:12:13.757752 31098 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 445. Preceding OpId from leader: term: 1 index: 446. (index mismatch)
I20240501 08:12:13.758265 2494 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:13.759163 2438 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:13.759619 2493 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:13.762110 30838 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 445. Preceding OpId from leader: term: 1 index: 446. (index mismatch)
I20240501 08:12:13.762673 2437 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 446, Last known committed idx: 445, Time since last communication: 0.000s
I20240501 08:12:13.764479 2437 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.765146 31098 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.765977 1837 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.769652 30561 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.770116 30838 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.770396 32293 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.773773 31227 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.446: config changed from index 445 to 446, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.777781 30965 consensus_queue.cc:237] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 446, Committed index: 446, Last appended: 1.446, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:13.781502 31227 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 446. Preceding OpId from leader: term: 1 index: 447. (index mismatch)
I20240501 08:12:13.782020 1837 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 446. Preceding OpId from leader: term: 1 index: 447. (index mismatch)
I20240501 08:12:13.782703 30838 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 446. Preceding OpId from leader: term: 1 index: 447. (index mismatch)
I20240501 08:12:13.783195 2495 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 447, Last known committed idx: 446, Time since last communication: 0.000s
I20240501 08:12:13.783939 2495 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 447, Last known committed idx: 446, Time since last communication: 0.000s
I20240501 08:12:13.784265 2493 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 447, Last known committed idx: 446, Time since last communication: 0.000s
I20240501 08:12:13.785849 31098 raft_consensus.cc:1261] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 57210dc0a07a4851a43be23a279eb836: Log matching property violated. Preceding OpId in replica: term: 1 index: 446. Preceding OpId from leader: term: 1 index: 447. (index mismatch)
I20240501 08:12:13.786847 2494 consensus_queue.cc:1035] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 447, Last known committed idx: 446, Time since last communication: 0.000s
I20240501 08:12:13.787415 2495 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 57210dc0a07a4851a43be23a279eb836 [term 1 LEADER]: Committing config change with OpId 1.447: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New config: { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.791869 1837 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.447: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New config: { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.792785 31227 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.447: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New config: { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.793566 31098 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.447: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New config: { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.793942 30552 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8302dd0a0d2e4a20bdc00e27da939e8a with cas_config_opid_index 446: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:13.795323 30840 raft_consensus.cc:2932] T 8302dd0a0d2e4a20bdc00e27da939e8a P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.447: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New config: { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.797552 30560 catalog_manager.cc:5484] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 reported cstate change: config changed from index 446 to 447, VOTER fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "57210dc0a07a4851a43be23a279eb836" committed_config { opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:13.827864 32280 tablet_service.cc:1514] Processing DeleteTablet for tablet 8302dd0a0d2e4a20bdc00e27da939e8a with delete_type TABLET_DATA_TOMBSTONED (TS fe99ab4507a24af4ba7fa5b5278c5ac6 not found in new config with opid_index 447) from {username='slave'} at 127.0.0.1:34080
I20240501 08:12:13.828779 2518 tablet_replica.cc:330] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: stopping tablet replica
I20240501 08:12:13.829166 2518 raft_consensus.cc:2229] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:13.829524 2518 raft_consensus.cc:2258] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:13.831703 2518 ts_tablet_manager.cc:1875] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:13.837805 2518 ts_tablet_manager.cc:1888] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.446
I20240501 08:12:13.838174 2518 log.cc:1196] T 8302dd0a0d2e4a20bdc00e27da939e8a P fe99ab4507a24af4ba7fa5b5278c5ac6: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:13.839470 30554 catalog_manager.cc:4830] TS fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533): tablet 8302dd0a0d2e4a20bdc00e27da939e8a (table rebalance_test_table_3 [id=3881d7493a564e21be970e6b9216ac77]) successfully deleted
I20240501 08:12:14.071985 2225 raft_consensus.cc:1050] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:14.073103 2225 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 529, Committed index: 529, Last appended: 1.529, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:14.075523 30705 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 529. Preceding OpId from leader: term: 1 index: 530. (index mismatch)
I20240501 08:12:14.075646 31360 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 529. Preceding OpId from leader: term: 1 index: 530. (index mismatch)
I20240501 08:12:14.076529 2464 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 530, Last known committed idx: 529, Time since last communication: 0.000s
I20240501 08:12:14.077494 2464 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 530, Last known committed idx: 529, Time since last communication: 0.000s
I20240501 08:12:14.077587 30840 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 529. Preceding OpId from leader: term: 1 index: 530. (index mismatch)
I20240501 08:12:14.078269 2464 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 530, Last known committed idx: 529, Time since last communication: 0.000s
I20240501 08:12:14.082015 1837 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 529. Preceding OpId from leader: term: 1 index: 530. (index mismatch)
I20240501 08:12:14.083096 31098 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 529. Preceding OpId from leader: term: 1 index: 530. (index mismatch)
I20240501 08:12:14.083786 2225 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 530, Last known committed idx: 529, Time since last communication: 0.000s
I20240501 08:12:14.084280 2535 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 530, Last known committed idx: 529, Time since last communication: 0.000s
I20240501 08:12:14.086472 2535 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.091316 30705 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.098296 1836 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.099874 30561 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:14.102102 30840 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.106899 31360 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.116812 31100 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.530: config changed from index 529 to 530, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 530 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } attrs { promote: false replace: true } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.119645 31227 consensus_queue.cc:237] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 530, Committed index: 530, Last appended: 1.530, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:14.121927 30840 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 530. Preceding OpId from leader: term: 1 index: 531. (index mismatch)
I20240501 08:12:14.122186 31360 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 530. Preceding OpId from leader: term: 1 index: 531. (index mismatch)
I20240501 08:12:14.122575 2225 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 531, Last known committed idx: 530, Time since last communication: 0.000s
I20240501 08:12:14.122649 1837 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 530. Preceding OpId from leader: term: 1 index: 531. (index mismatch)
I20240501 08:12:14.123168 2225 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 531, Last known committed idx: 530, Time since last communication: 0.000s
I20240501 08:12:14.123505 31096 raft_consensus.cc:1261] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 9204a82012ac492dbc0da9c84c51dabf: Log matching property violated. Preceding OpId in replica: term: 1 index: 530. Preceding OpId from leader: term: 1 index: 531. (index mismatch)
I20240501 08:12:14.125639 2225 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 531, Last known committed idx: 530, Time since last communication: 0.000s
I20240501 08:12:14.125890 2535 consensus_queue.cc:1035] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 531, Last known committed idx: 530, Time since last communication: 0.000s
I20240501 08:12:14.127563 2535 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf [term 1 LEADER]: Committing config change with OpId 1.531: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.128705 31360 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.531: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.132872 30561 catalog_manager.cc:5484] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf reported cstate change: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "9204a82012ac492dbc0da9c84c51dabf" committed_config { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:14.133466 30553 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5730faa05092410da14e00e24ad74c86 with cas_config_opid_index 530: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:14.134958 31098 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.531: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.136025 1837 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.531: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.137809 30840 raft_consensus.cc:2932] T 5730faa05092410da14e00e24ad74c86 P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Committing config change with OpId 1.531: config changed from index 530 to 531, VOTER 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129) evicted. New config: { opid_index: 531 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.141263 30687 tablet_service.cc:1514] Processing DeleteTablet for tablet 5730faa05092410da14e00e24ad74c86 with delete_type TABLET_DATA_TOMBSTONED (TS 3170be3b2bbb4b3bbd8a37ac5f91de89 not found in new config with opid_index 531) from {username='slave'} at 127.0.0.1:43428
I20240501 08:12:14.141901 2545 tablet_replica.cc:330] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: stopping tablet replica
I20240501 08:12:14.142189 2545 raft_consensus.cc:2229] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:14.144646 2545 raft_consensus.cc:2258] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:14.146746 1837 raft_consensus.cc:1203] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.407->[1.408-1.408] Dedup: 1.408->[]
I20240501 08:12:14.147195 2545 ts_tablet_manager.cc:1875] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:14.152699 2545 ts_tablet_manager.cc:1888] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.530
I20240501 08:12:14.152838 2545 log.cc:1196] T 5730faa05092410da14e00e24ad74c86 P 3170be3b2bbb4b3bbd8a37ac5f91de89: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/5730faa05092410da14e00e24ad74c86
I20240501 08:12:14.153455 30552 catalog_manager.cc:4830] TS 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): tablet 5730faa05092410da14e00e24ad74c86 (table rebalance_test_table_1 [id=e93d8483bead454d8e47e99f73d1dabb]) successfully deleted
I20240501 08:12:14.692680 2546 raft_consensus.cc:1050] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:14.693940 2546 consensus_queue.cc:237] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 408, Committed index: 408, Last appended: 1.408, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:14.696563 1837 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 408. Preceding OpId from leader: term: 1 index: 409. (index mismatch)
I20240501 08:12:14.697731 31227 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 408. Preceding OpId from leader: term: 1 index: 409. (index mismatch)
I20240501 08:12:14.699330 2569 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 409, Last known committed idx: 408, Time since last communication: 0.000s
I20240501 08:12:14.699941 2569 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 409, Last known committed idx: 408, Time since last communication: 0.000s
I20240501 08:12:14.705919 31360 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 408. Preceding OpId from leader: term: 1 index: 409. (index mismatch)
I20240501 08:12:14.706419 30705 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 408. Preceding OpId from leader: term: 1 index: 409. (index mismatch)
I20240501 08:12:14.706714 30965 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 408. Preceding OpId from leader: term: 1 index: 409. (index mismatch)
I20240501 08:12:14.706756 2570 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 409, Last known committed idx: 408, Time since last communication: 0.000s
I20240501 08:12:14.707046 2573 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 409, Last known committed idx: 408, Time since last communication: 0.000s
I20240501 08:12:14.707607 2546 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 409, Last known committed idx: 408, Time since last communication: 0.000s
I20240501 08:12:14.710384 2570 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.710932 31360 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.712404 30965 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.712620 30705 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.715237 31227 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.716809 30561 catalog_manager.cc:5484] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade reported cstate change: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.721222 1837 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.409: config changed from index 408 to 409, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 409 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } attrs { replace: true } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.733434 30840 consensus_queue.cc:237] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 409, Committed index: 409, Last appended: 1.409, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:14.735818 31360 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 409. Preceding OpId from leader: term: 1 index: 410. (index mismatch)
I20240501 08:12:14.736018 1837 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 409. Preceding OpId from leader: term: 1 index: 410. (index mismatch)
I20240501 08:12:14.736204 30965 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 409. Preceding OpId from leader: term: 1 index: 410. (index mismatch)
I20240501 08:12:14.736462 30705 raft_consensus.cc:1261] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 409. Preceding OpId from leader: term: 1 index: 410. (index mismatch)
I20240501 08:12:14.736416 2546 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 410, Last known committed idx: 409, Time since last communication: 0.000s
I20240501 08:12:14.737205 2572 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 410, Last known committed idx: 409, Time since last communication: 0.000s
I20240501 08:12:14.737812 2572 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 410, Last known committed idx: 409, Time since last communication: 0.000s
I20240501 08:12:14.738371 2573 consensus_queue.cc:1035] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 410, Last known committed idx: 409, Time since last communication: 0.000s
I20240501 08:12:14.740495 2572 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.410: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.741246 30965 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 57210dc0a07a4851a43be23a279eb836 [term 1 FOLLOWER]: Committing config change with OpId 1.410: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.741762 30705 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 [term 1 FOLLOWER]: Committing config change with OpId 1.410: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.746150 30563 catalog_manager.cc:5484] T 3c02021c881348748fa99b0df2dfb228 P 3170be3b2bbb4b3bbd8a37ac5f91de89 reported cstate change: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.746057 31360 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.410: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.747575 30555 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3c02021c881348748fa99b0df2dfb228 with cas_config_opid_index 409: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:14.744580 1837 raft_consensus.cc:2932] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.410: config changed from index 409 to 410, VOTER 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133) evicted. New config: { opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:14.753475 31209 tablet_service.cc:1514] Processing DeleteTablet for tablet 3c02021c881348748fa99b0df2dfb228 with delete_type TABLET_DATA_TOMBSTONED (TS 9204a82012ac492dbc0da9c84c51dabf not found in new config with opid_index 410) from {username='slave'} at 127.0.0.1:57760
I20240501 08:12:14.754935 2585 tablet_replica.cc:330] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: stopping tablet replica
I20240501 08:12:14.755106 2585 raft_consensus.cc:2229] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:14.755340 2585 raft_consensus.cc:2258] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:14.757107 2585 ts_tablet_manager.cc:1875] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:14.761451 2585 ts_tablet_manager.cc:1888] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.409
I20240501 08:12:14.761540 2585 log.cc:1196] T 3c02021c881348748fa99b0df2dfb228 P 9204a82012ac492dbc0da9c84c51dabf: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/3c02021c881348748fa99b0df2dfb228
I20240501 08:12:14.762221 30553 catalog_manager.cc:4830] TS 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): tablet 3c02021c881348748fa99b0df2dfb228 (table rebalance_test_table_4 [id=9d6e5a2902824bacbbd466c9e3ece3b1]) successfully deleted
I20240501 08:12:14.909516 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:14.921733 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:14.922856 30818 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:14.931182 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:14.940102 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:15.005000 32280 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:15.005880 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:15.018764 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:15.127640 30840 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 326, Committed index: 326, Last appended: 1.326, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:15.130399 31360 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 326. Preceding OpId from leader: term: 1 index: 327. (index mismatch)
I20240501 08:12:15.130699 32293 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 326. Preceding OpId from leader: term: 1 index: 327. (index mismatch)
I20240501 08:12:15.130941 31224 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 326. Preceding OpId from leader: term: 1 index: 327. (index mismatch)
I20240501 08:12:15.131011 2572 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 327, Last known committed idx: 326, Time since last communication: 0.000s
I20240501 08:12:15.131518 2572 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 327, Last known committed idx: 326, Time since last communication: 0.000s
I20240501 08:12:15.132419 31098 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 326. Preceding OpId from leader: term: 1 index: 327. (index mismatch)
I20240501 08:12:15.133786 2573 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 327, Last known committed idx: 326, Time since last communication: 0.000s
I20240501 08:12:15.134073 2572 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 327, Last known committed idx: 326, Time since last communication: 0.000s
W20240501 08:12:15.134346 30774 consensus_peers.cc:485] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 -> Peer 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857): Couldn't send request to peer 096aec539f1541b4b3049afbdcf36a81. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 568f82bc574e4c6fafe9ddb3375429ae. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:15.134707 2570 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.327: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:15.135856 32293 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.327: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:15.137476 31224 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.327: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:15.141268 31360 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.327: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:15.142392 30563 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:15.163182 31098 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.327: config changed from index 204 to 327, NON_VOTER 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) added. New config: { opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } } }
I20240501 08:12:15.601131 2659 ts_tablet_manager.cc:902] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Initiating tablet copy from peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291)
I20240501 08:12:15.602191 2659 tablet_copy_client.cc:308] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:46291
I20240501 08:12:15.602814 30860 tablet_copy_service.cc:140] P 1a388864321846929919a37b570e8138: Received BeginTabletCopySession request for tablet 568f82bc574e4c6fafe9ddb3375429ae from peer 096aec539f1541b4b3049afbdcf36a81 ({username='slave'} at 127.28.1.136:52233)
I20240501 08:12:15.602955 30860 tablet_copy_service.cc:161] P 1a388864321846929919a37b570e8138: Beginning new tablet copy session on tablet 568f82bc574e4c6fafe9ddb3375429ae from peer 096aec539f1541b4b3049afbdcf36a81 at {username='slave'} at 127.28.1.136:52233: session id = 096aec539f1541b4b3049afbdcf36a81-568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:15.605170 30860 tablet_copy_source_session.cc:205] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:15.606307 2659 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 568f82bc574e4c6fafe9ddb3375429ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:15.610667 2659 tablet_copy_client.cc:783] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:15.610875 2659 tablet_copy_client.cc:647] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:15.612922 2659 tablet_copy_client.cc:523] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:15.615173 2659 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Bootstrap starting.
I20240501 08:12:15.899168 2659 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Bootstrap replayed 1/1 log segments. Stats: ops{read=327 overwritten=0 applied=327 ignored=0} inserts{seen=996 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:15.899531 2659 tablet_bootstrap.cc:492] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Bootstrap complete.
I20240501 08:12:15.899798 2659 ts_tablet_manager.cc:1367] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Time spent bootstrapping tablet: real 0.285s user 0.252s sys 0.031s
I20240501 08:12:15.900444 2659 raft_consensus.cc:348] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:15.900656 2659 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 096aec539f1541b4b3049afbdcf36a81, State: Initialized, Role: LEARNER
I20240501 08:12:15.900837 2659 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 327, Last appended: 1.327, Last appended by leader: 327, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 327 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: true } }
I20240501 08:12:15.901486 2659 ts_tablet_manager.cc:1398] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81: Time spent starting tablet: real 0.002s user 0.001s sys 0.004s
I20240501 08:12:15.902396 30860 tablet_copy_service.cc:342] P 1a388864321846929919a37b570e8138: Request end of tablet copy session 096aec539f1541b4b3049afbdcf36a81-568f82bc574e4c6fafe9ddb3375429ae received from {username='slave'} at 127.28.1.136:52233
I20240501 08:12:15.902527 30860 tablet_copy_service.cc:433] P 1a388864321846929919a37b570e8138: ending tablet copy session 096aec539f1541b4b3049afbdcf36a81-568f82bc574e4c6fafe9ddb3375429ae on tablet 568f82bc574e4c6fafe9ddb3375429ae with peer 096aec539f1541b4b3049afbdcf36a81
I20240501 08:12:16.197202 1837 raft_consensus.cc:1203] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.326->[1.327-1.327] Dedup: 1.327->[]
I20240501 08:12:16.698262 2695 raft_consensus.cc:1050] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: attempting to promote NON_VOTER 096aec539f1541b4b3049afbdcf36a81 to VOTER
I20240501 08:12:16.698984 2695 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 327, Committed index: 327, Last appended: 1.327, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:16.701606 31224 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240501 08:12:16.701625 1837 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 LEARNER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240501 08:12:16.702169 2697 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240501 08:12:16.702437 32293 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240501 08:12:16.702535 2678 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240501 08:12:16.703053 2678 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240501 08:12:16.715853 31360 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240501 08:12:16.715844 31098 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Refusing update from remote peer 1a388864321846929919a37b570e8138: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240501 08:12:16.716435 2697 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240501 08:12:16.716727 2678 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240501 08:12:16.720036 2678 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.720927 31360 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.724854 32293 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.725735 30567 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 reported cstate change: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1a388864321846929919a37b570e8138" committed_config { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:16.728307 1836 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.732478 31224 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.732443 31100 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index 327 to 328, 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:16.864331 30840 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae"
dest_uuid: "1a388864321846929919a37b570e8138"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:58630
I20240501 08:12:16.864568 30840 raft_consensus.cc:595] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:12:17.080790 30840 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae"
dest_uuid: "1a388864321846929919a37b570e8138"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:58650
I20240501 08:12:17.080950 30840 raft_consensus.cc:595] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:12:17.113520 2698 raft_consensus.cc:979] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: : Instructing follower fe99ab4507a24af4ba7fa5b5278c5ac6 to start an election
I20240501 08:12:17.113677 2704 raft_consensus.cc:1067] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Signalling peer fe99ab4507a24af4ba7fa5b5278c5ac6 to start an election
I20240501 08:12:17.114511 32293 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae"
dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6"
from {username='slave'} at 127.28.1.130:40305
I20240501 08:12:17.114748 32293 raft_consensus.cc:482] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:12:17.114885 32293 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.117158 32293 raft_consensus.cc:504] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:17.119318 31098 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 328 } } ignore_live_leader: true dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4"
I20240501 08:12:17.119534 31098 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.121920 32293 leader_election.cc:288] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 election: Requested vote from peers 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857)
I20240501 08:12:17.122373 31098 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 2.
I20240501 08:12:17.123006 30840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 328 } } ignore_live_leader: true dest_uuid: "1a388864321846929919a37b570e8138"
I20240501 08:12:17.123205 30840 raft_consensus.cc:3032] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:12:17.123270 30840 raft_consensus.cc:729] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 1a388864321846929919a37b570e8138, State: Running, Role: LEADER
I20240501 08:12:17.123183 31360 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 328 } } ignore_live_leader: true dest_uuid: "6f37f9ed957342dba1271066c1bbeade"
I20240501 08:12:17.123410 31360 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.123584 30840 consensus_queue.cc:260] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 328, Committed index: 328, Last appended: 1.328, Last appended by leader: 328, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:17.124054 30840 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.125969 30840 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 2.
I20240501 08:12:17.126327 31360 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 2.
I20240501 08:12:17.126857 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 328 } } ignore_live_leader: true dest_uuid: "9204a82012ac492dbc0da9c84c51dabf"
I20240501 08:12:17.127099 31224 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.130142 31224 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 2.
I20240501 08:12:17.130934 32232 leader_election.cc:302] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 6 voters: 4 yes votes; 0 no votes. yes voters: 1a388864321846929919a37b570e8138, 6f37f9ed957342dba1271066c1bbeade, 9204a82012ac492dbc0da9c84c51dabf, fe99ab4507a24af4ba7fa5b5278c5ac6; no voters:
I20240501 08:12:17.131326 2707 raft_consensus.cc:2781] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:12:17.132194 2707 raft_consensus.cc:686] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 LEADER]: Becoming Leader. State: Replica: fe99ab4507a24af4ba7fa5b5278c5ac6, State: Running, Role: LEADER
I20240501 08:12:17.132663 2707 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 328, Committed index: 328, Last appended: 1.328, Last appended by leader: 328, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:17.136454 30566 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 reported cstate change: term changed from 1 to 2, leader changed from 1a388864321846929919a37b570e8138 (127.28.1.130) to fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135). New cstate: current_term: 2 leader_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" committed_config { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:17.137789 1836 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "568f82bc574e4c6fafe9ddb3375429ae" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 328 } } ignore_live_leader: true dest_uuid: "096aec539f1541b4b3049afbdcf36a81"
I20240501 08:12:17.138116 1836 raft_consensus.cc:3037] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:17.140368 1836 raft_consensus.cc:2454] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 2.
I20240501 08:12:17.661386 31360 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 1 index: 328. Preceding OpId from leader: term: 2 index: 329. (index mismatch)
I20240501 08:12:17.661996 2746 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 329, Last known committed idx: 328, Time since last communication: 0.000s
I20240501 08:12:17.689597 30840 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 1 index: 328. Preceding OpId from leader: term: 2 index: 329. (index mismatch)
I20240501 08:12:17.690120 2746 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 329, Last known committed idx: 328, Time since last communication: 0.000s
I20240501 08:12:17.706141 1836 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 1 index: 328. Preceding OpId from leader: term: 2 index: 329. (index mismatch)
I20240501 08:12:17.706692 2746 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 329, Last known committed idx: 328, Time since last communication: 0.000s
I20240501 08:12:17.711480 31224 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 1 index: 328. Preceding OpId from leader: term: 2 index: 329. (index mismatch)
I20240501 08:12:17.712474 2746 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 329, Last known committed idx: 328, Time since last communication: 0.000s
I20240501 08:12:17.722770 31098 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 1 index: 328. Preceding OpId from leader: term: 2 index: 329. (index mismatch)
I20240501 08:12:17.724390 2759 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 329, Last known committed idx: 328, Time since last communication: 0.000s
I20240501 08:12:18.300453 32293 consensus_queue.cc:237] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 329, Committed index: 329, Last appended: 2.329, Last appended by leader: 328, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:18.302477 31098 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 2 index: 329. Preceding OpId from leader: term: 2 index: 330. (index mismatch)
I20240501 08:12:18.302518 31224 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 2 index: 329. Preceding OpId from leader: term: 2 index: 330. (index mismatch)
I20240501 08:12:18.302625 1836 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 2 index: 329. Preceding OpId from leader: term: 2 index: 330. (index mismatch)
I20240501 08:12:18.303000 2755 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 330, Last known committed idx: 329, Time since last communication: 0.000s
I20240501 08:12:18.303378 2753 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 330, Last known committed idx: 329, Time since last communication: 0.000s
I20240501 08:12:18.303504 31360 raft_consensus.cc:1261] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Refusing update from remote peer fe99ab4507a24af4ba7fa5b5278c5ac6: Log matching property violated. Preceding OpId in replica: term: 2 index: 329. Preceding OpId from leader: term: 2 index: 330. (index mismatch)
I20240501 08:12:18.303735 2759 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 330, Last known committed idx: 329, Time since last communication: 0.000s
I20240501 08:12:18.304247 2755 consensus_queue.cc:1035] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 330, Last known committed idx: 329, Time since last communication: 0.000s
I20240501 08:12:18.307466 2759 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 2 LEADER]: Committing config change with OpId 2.330: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.307881 31098 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 [term 2 FOLLOWER]: Committing config change with OpId 2.330: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.307994 1836 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Committing config change with OpId 2.330: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.313215 31224 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 9204a82012ac492dbc0da9c84c51dabf [term 2 FOLLOWER]: Committing config change with OpId 2.330: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.315034 30559 catalog_manager.cc:5484] T 568f82bc574e4c6fafe9ddb3375429ae P 9fc0640dbc0d4208bfb536949d1bb0c4 reported cstate change: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New cstate: current_term: 2 leader_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" committed_config { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.318487 30554 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 568f82bc574e4c6fafe9ddb3375429ae with cas_config_opid_index 328: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:18.321413 31360 raft_consensus.cc:2932] T 568f82bc574e4c6fafe9ddb3375429ae P 6f37f9ed957342dba1271066c1bbeade [term 2 FOLLOWER]: Committing config change with OpId 2.330: config changed from index 328 to 330, VOTER 1a388864321846929919a37b570e8138 (127.28.1.130) evicted. New config: { opid_index: 330 OBSOLETE_local: false peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } } }
I20240501 08:12:18.324992 30818 tablet_service.cc:1514] Processing DeleteTablet for tablet 568f82bc574e4c6fafe9ddb3375429ae with delete_type TABLET_DATA_TOMBSTONED (TS 1a388864321846929919a37b570e8138 not found in new config with opid_index 330) from {username='slave'} at 127.0.0.1:37872
I20240501 08:12:18.328650 2780 tablet_replica.cc:330] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: stopping tablet replica
I20240501 08:12:18.328859 2780 raft_consensus.cc:2229] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:18.329038 2780 raft_consensus.cc:2258] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:18.330463 2780 ts_tablet_manager.cc:1875] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:18.336738 2780 ts_tablet_manager.cc:1888] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.329
I20240501 08:12:18.336846 2780 log.cc:1196] T 568f82bc574e4c6fafe9ddb3375429ae P 1a388864321846929919a37b570e8138: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:18.337625 30555 catalog_manager.cc:4830] TS 1a388864321846929919a37b570e8138 (127.28.1.130:46291): tablet 568f82bc574e4c6fafe9ddb3375429ae (table rebalance_test_table_2 [id=611625662521483486949c0824ba78a9]) successfully deleted
I20240501 08:12:18.484115 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:18.487854 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.506731 30818 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.520413 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:18.534284 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:18.593266 32280 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:18.597959 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:18.628183 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.782802 30818 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.784411 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:18.799698 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.812633 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:18.825472 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:18.828603 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:18.829468 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:18.864960 32280 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:19.243759 30818 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:19.243993 1817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:19.254601 30687 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:19.259747 31338 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:12:19.282028 31209 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:19.286931 30948 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:12:19.323777 31079 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:12:19.351631 32280 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
b5013b5f37d84aa1810fafbf49ae9861 | 127.28.1.190:35983 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:41967 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
096aec539f1541b4b3049afbdcf36a81 | 127.28.1.136:43857 | HEALTHY | <none> | 0 | 0
1a388864321846929919a37b570e8138 | 127.28.1.130:46291 | HEALTHY | <none> | 2 | 0
3170be3b2bbb4b3bbd8a37ac5f91de89 | 127.28.1.129:39637 | HEALTHY | <none> | 2 | 0
57210dc0a07a4851a43be23a279eb836 | 127.28.1.131:39451 | HEALTHY | <none> | 3 | 0
6f37f9ed957342dba1271066c1bbeade | 127.28.1.134:41579 | HEALTHY | <none> | 1 | 0
9204a82012ac492dbc0da9c84c51dabf | 127.28.1.133:42523 | HEALTHY | <none> | 5 | 0
9fc0640dbc0d4208bfb536949d1bb0c4 | 127.28.1.132:41135 | HEALTHY | <none> | 2 | 0
fe99ab4507a24af4ba7fa5b5278c5ac6 | 127.28.1.135:36533 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 8
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 8 server(s) checked
local_ip_for_outbound_sockets | 127.28.1.129 | experimental | 127.28.1.129:39637
local_ip_for_outbound_sockets | 127.28.1.130 | experimental | 127.28.1.130:46291
local_ip_for_outbound_sockets | 127.28.1.131 | experimental | 127.28.1.131:39451
local_ip_for_outbound_sockets | 127.28.1.132 | experimental | 127.28.1.132:41135
local_ip_for_outbound_sockets | 127.28.1.133 | experimental | 127.28.1.133:42523
local_ip_for_outbound_sockets | 127.28.1.134 | experimental | 127.28.1.134:41579
local_ip_for_outbound_sockets | 127.28.1.135 | experimental | 127.28.1.135:36533
local_ip_for_outbound_sockets | 127.28.1.136 | experimental | 127.28.1.136:43857
never_fsync | true | unsafe,advanced | all 8 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 8 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 8 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 8 server(s) checked
server_dump_info_format | pb | hidden | all 8 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.28.1.129:39637
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.28.1.130:46291
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.28.1.131:39451
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.28.1.132:41135
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.28.1.133:42523
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.28.1.134:41579
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.28.1.135:36533
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.28.1.136:43857
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:41967 | all 8 server(s) checked
time_source | builtin | all 8 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 9 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 8
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240501 08:12:19.425652 28678 log_verifier.cc:126] Checking tablet 120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:12:19.870007 28678 log_verifier.cc:177] Verified matching terms for 508 ops in tablet 120d5a7fb6704b4eb07c02ab8dabf6cd
I20240501 08:12:19.871088 28678 log_verifier.cc:126] Checking tablet 1280b2f88bbc470bbc9bc49c95a4b108
I20240501 08:12:20.191869 28678 log_verifier.cc:177] Verified matching terms for 372 ops in tablet 1280b2f88bbc470bbc9bc49c95a4b108
I20240501 08:12:20.192812 28678 log_verifier.cc:126] Checking tablet 1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:20.222237 28678 log_verifier.cc:177] Verified matching terms for 4 ops in tablet 1469837fa8424b94b9ec4bbe57fdf740
I20240501 08:12:20.222335 28678 log_verifier.cc:126] Checking tablet 3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:20.536301 28678 log_verifier.cc:177] Verified matching terms for 356 ops in tablet 3b8a23ca81da4039b7c18c84b2f2d8fb
I20240501 08:12:20.537436 28678 log_verifier.cc:126] Checking tablet 3c02021c881348748fa99b0df2dfb228
I20240501 08:12:20.891269 28678 log_verifier.cc:177] Verified matching terms for 410 ops in tablet 3c02021c881348748fa99b0df2dfb228
I20240501 08:12:20.892298 28678 log_verifier.cc:126] Checking tablet 568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:21.189133 28678 log_verifier.cc:177] Verified matching terms for 330 ops in tablet 568f82bc574e4c6fafe9ddb3375429ae
I20240501 08:12:21.189898 28678 log_verifier.cc:126] Checking tablet 5730faa05092410da14e00e24ad74c86
I20240501 08:12:21.630870 28678 log_verifier.cc:177] Verified matching terms for 531 ops in tablet 5730faa05092410da14e00e24ad74c86
I20240501 08:12:21.632126 28678 log_verifier.cc:126] Checking tablet 6920f9b58f814b69a3cab12f5fa3de91
I20240501 08:12:21.937368 28678 log_verifier.cc:177] Verified matching terms for 375 ops in tablet 6920f9b58f814b69a3cab12f5fa3de91
I20240501 08:12:21.938257 28678 log_verifier.cc:126] Checking tablet 8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:22.335214 28678 log_verifier.cc:177] Verified matching terms for 447 ops in tablet 8302dd0a0d2e4a20bdc00e27da939e8a
I20240501 08:12:22.336231 28678 log_verifier.cc:126] Checking tablet 9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:12:22.725834 28678 log_verifier.cc:177] Verified matching terms for 459 ops in tablet 9c025c9ec2c74a54ab24c40ed11639f6
I20240501 08:12:22.726857 28678 log_verifier.cc:126] Checking tablet 9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:23.086529 28678 log_verifier.cc:177] Verified matching terms for 446 ops in tablet 9ff3c78386224499b7022dc9334ddd33
I20240501 08:12:23.087924 28678 log_verifier.cc:126] Checking tablet a465d27916734303bd9d1bed2a3dfb14
I20240501 08:12:23.430922 28678 log_verifier.cc:177] Verified matching terms for 390 ops in tablet a465d27916734303bd9d1bed2a3dfb14
I20240501 08:12:23.431807 28678 log_verifier.cc:126] Checking tablet b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:23.777931 28678 log_verifier.cc:177] Verified matching terms for 377 ops in tablet b3a5e3d4cc604fcd90a0efeea3946acd
I20240501 08:12:23.778894 28678 log_verifier.cc:126] Checking tablet bc20191679084ecfbd360e23d68abd62
I20240501 08:12:24.091490 28678 log_verifier.cc:177] Verified matching terms for 382 ops in tablet bc20191679084ecfbd360e23d68abd62
I20240501 08:12:24.092617 28678 log_verifier.cc:126] Checking tablet e2393202bb014986b2496d3c99c65962
I20240501 08:12:24.413949 28678 log_verifier.cc:177] Verified matching terms for 354 ops in tablet e2393202bb014986b2496d3c99c65962
I20240501 08:12:24.414672 28678 log_verifier.cc:126] Checking tablet e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:24.819911 28678 log_verifier.cc:177] Verified matching terms for 502 ops in tablet e61aa5646d3947c7b68bef19ee280c6e
I20240501 08:12:24.832582 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30627
I20240501 08:12:24.851027 30748 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:25.136451 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30627
W20240501 08:12:25.176123 32234 connection.cc:535] server connection from 127.28.1.129:45449 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.176441 31163 connection.cc:535] server connection from 127.28.1.129:57131 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.176632 31163 connection.cc:535] client connection to 127.28.1.129:39637 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.176712 31163 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 2 similar messages]
W20240501 08:12:25.176787 31033 connection.cc:535] client connection to 127.28.1.129:39637 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:12:25.176930 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30758
W20240501 08:12:25.177024 31033 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.177141 31034 connection.cc:535] server connection from 127.28.1.129:58969 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.179522 31033 consensus_peers.cc:485] T 1280b2f88bbc470bbc9bc49c95a4b108 P 9fc0640dbc0d4208bfb536949d1bb0c4 -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:25.179873 31163 consensus_peers.cc:485] T e61aa5646d3947c7b68bef19ee280c6e P 9204a82012ac492dbc0da9c84c51dabf -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:25.183626 30880 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:12:25.241299 30903 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
W20240501 08:12:25.242527 30903 consensus_peers.cc:485] T 9c025c9ec2c74a54ab24c40ed11639f6 P 57210dc0a07a4851a43be23a279eb836 -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:25.320566 31163 consensus_peers.cc:485] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:25.469048 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30758
W20240501 08:12:25.508327 30905 connection.cc:535] server connection from 127.28.1.130:49241 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:25.508358 31293 connection.cc:535] server connection from 127.28.1.130:37091 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:12:25.509274 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30888
W20240501 08:12:25.510221 31163 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:25.514379 31009 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:12:25.521823 31166 consensus_peers.cc:485] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:25.583544 31033 consensus_peers.cc:485] T 9ff3c78386224499b7022dc9334ddd33 P 9fc0640dbc0d4208bfb536949d1bb0c4 -> Peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Couldn't send request to peer 3170be3b2bbb4b3bbd8a37ac5f91de89. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:25.792440 31166 consensus_peers.cc:485] T 1469837fa8424b94b9ec4bbe57fdf740 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:25.794349 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30888
W20240501 08:12:25.801535 31296 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111)
W20240501 08:12:25.802626 31296 consensus_peers.cc:485] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:25.836387 31296 connection.cc:535] server connection from 127.28.1.131:34347 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:12:25.836731 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 31018
W20240501 08:12:25.837651 31293 consensus_peers.cc:485] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade -> Peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Couldn't send request to peer 57210dc0a07a4851a43be23a279eb836. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.131:39451: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:25.842444 31139 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:12:25.906015 31166 consensus_peers.cc:485] T bc20191679084ecfbd360e23d68abd62 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Couldn't send request to peer 1a388864321846929919a37b570e8138. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:26.080264 2929 raft_consensus.cc:482] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3170be3b2bbb4b3bbd8a37ac5f91de89)
I20240501 08:12:26.080435 2929 raft_consensus.cc:504] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:26.081497 2929 leader_election.cc:288] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533), 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857)
I20240501 08:12:26.082114 32293 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" is_pre_election: true
I20240501 08:12:26.082190 1837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "096aec539f1541b4b3049afbdcf36a81" is_pre_election: true
W20240501 08:12:26.084170 31163 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
I20240501 08:12:26.084275 31163 leader_election.cc:302] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 9204a82012ac492dbc0da9c84c51dabf; no voters: 096aec539f1541b4b3049afbdcf36a81, 3170be3b2bbb4b3bbd8a37ac5f91de89, fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:12:26.084533 2929 raft_consensus.cc:2726] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240501 08:12:26.099179 31163 consensus_peers.cc:485] T 3b8a23ca81da4039b7c18c84b2f2d8fb P 9204a82012ac492dbc0da9c84c51dabf -> Peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Couldn't send request to peer 57210dc0a07a4851a43be23a279eb836. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.131:39451: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:26.118271 31098 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "9204a82012ac492dbc0da9c84c51dabf" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" is_pre_election: true
I20240501 08:12:26.119622 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 31018
W20240501 08:12:26.163091 31164 connection.cc:535] client connection to 127.28.1.132:41135 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:12:26.163211 31294 connection.cc:535] client connection to 127.28.1.132:41135 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:12:26.163654 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 31148
W20240501 08:12:26.164271 31164 consensus_peers.cc:485] T 5730faa05092410da14e00e24ad74c86 P 9204a82012ac492dbc0da9c84c51dabf -> Peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Couldn't send request to peer 9fc0640dbc0d4208bfb536949d1bb0c4. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:12:26.164322 31294 consensus_peers.cc:485] T 120d5a7fb6704b4eb07c02ab8dabf6cd P 6f37f9ed957342dba1271066c1bbeade -> Peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Couldn't send request to peer 9fc0640dbc0d4208bfb536949d1bb0c4. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:26.168900 31270 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:12:26.196758 32232 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
W20240501 08:12:26.198086 32232 consensus_peers.cc:485] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 -> Peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Couldn't send request to peer 9fc0640dbc0d4208bfb536949d1bb0c4. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:26.207739 2943 raft_consensus.cc:482] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3170be3b2bbb4b3bbd8a37ac5f91de89)
I20240501 08:12:26.207923 2943 raft_consensus.cc:504] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } }
I20240501 08:12:26.208905 2943 leader_election.cc:288] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
W20240501 08:12:26.211238 31293 leader_election.cc:334] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
W20240501 08:12:26.211743 31293 leader_election.cc:334] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Network error: Client connection negotiation failed: client connection to 127.28.1.131:39451: connect: Connection refused (error 111)
W20240501 08:12:26.212962 31294 leader_election.cc:334] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
I20240501 08:12:26.213050 31294 leader_election.cc:302] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 6f37f9ed957342dba1271066c1bbeade; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:12:26.213299 2943 raft_consensus.cc:2726] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:12:26.350952 2949 raft_consensus.cc:482] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3170be3b2bbb4b3bbd8a37ac5f91de89)
I20240501 08:12:26.351174 2949 raft_consensus.cc:504] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:26.353139 2949 leader_election.cc:288] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), fe99ab4507a24af4ba7fa5b5278c5ac6 (127.28.1.135:36533)
W20240501 08:12:26.354583 1771 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
W20240501 08:12:26.365247 1771 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
W20240501 08:12:26.367324 1770 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
I20240501 08:12:26.370997 32293 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "096aec539f1541b4b3049afbdcf36a81" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" is_pre_election: true
I20240501 08:12:26.371598 1772 leader_election.cc:302] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 096aec539f1541b4b3049afbdcf36a81; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 9fc0640dbc0d4208bfb536949d1bb0c4, fe99ab4507a24af4ba7fa5b5278c5ac6
I20240501 08:12:26.371944 2949 raft_consensus.cc:2726] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:12:26.458941 31227 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "096aec539f1541b4b3049afbdcf36a81" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "9204a82012ac492dbc0da9c84c51dabf" is_pre_election: true
I20240501 08:12:26.458998 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 31148
I20240501 08:12:26.459151 31227 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 9204a82012ac492dbc0da9c84c51dabf [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 096aec539f1541b4b3049afbdcf36a81 in term 1.
I20240501 08:12:26.500514 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 31278
W20240501 08:12:26.501170 1771 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Network error: Client connection negotiation failed: client connection to 127.28.1.133:42523: connect: Connection refused (error 111)
W20240501 08:12:26.501516 31294 leader_election.cc:334] T 6920f9b58f814b69a3cab12f5fa3de91 P 6f37f9ed957342dba1271066c1bbeade [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Network error: Client connection negotiation failed: client connection to 127.28.1.133:42523: connect: Connection refused (error 111)
I20240501 08:12:26.506006 31400 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:26.544121 2949 raft_consensus.cc:482] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1a388864321846929919a37b570e8138)
I20240501 08:12:26.544330 2949 raft_consensus.cc:504] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 410 OBSOLETE_local: false peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:26.545372 2949 leader_election.cc:288] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
W20240501 08:12:26.546669 1773 leader_election.cc:334] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111)
W20240501 08:12:26.546875 1770 leader_election.cc:334] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
W20240501 08:12:26.547086 1770 leader_election.cc:334] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Network error: Client connection negotiation failed: client connection to 127.28.1.131:39451: connect: Connection refused (error 111)
I20240501 08:12:26.547173 1770 leader_election.cc:302] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 096aec539f1541b4b3049afbdcf36a81; no voters: 1a388864321846929919a37b570e8138, 3170be3b2bbb4b3bbd8a37ac5f91de89, 57210dc0a07a4851a43be23a279eb836
I20240501 08:12:26.547417 2949 raft_consensus.cc:2726] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:12:26.595199 2949 raft_consensus.cc:482] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57210dc0a07a4851a43be23a279eb836)
I20240501 08:12:26.595439 2949 raft_consensus.cc:504] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 447 OBSOLETE_local: false peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:26.596521 2949 leader_election.cc:288] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523)
W20240501 08:12:26.598040 1773 leader_election.cc:334] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1a388864321846929919a37b570e8138 (127.28.1.130:46291): Network error: Client connection negotiation failed: client connection to 127.28.1.130:46291: connect: Connection refused (error 111)
W20240501 08:12:26.598487 1770 leader_election.cc:334] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451): Network error: Client connection negotiation failed: client connection to 127.28.1.131:39451: connect: Connection refused (error 111)
W20240501 08:12:26.598887 1771 leader_election.cc:334] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
I20240501 08:12:26.599027 1771 leader_election.cc:302] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 096aec539f1541b4b3049afbdcf36a81; no voters: 1a388864321846929919a37b570e8138, 57210dc0a07a4851a43be23a279eb836, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:12:26.599272 2949 raft_consensus.cc:2726] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240501 08:12:26.599671 1771 leader_election.cc:334] T 8302dd0a0d2e4a20bdc00e27da939e8a P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Network error: Client connection negotiation failed: client connection to 127.28.1.133:42523: connect: Connection refused (error 111)
W20240501 08:12:26.625777 32232 consensus_peers.cc:485] T 568f82bc574e4c6fafe9ddb3375429ae P fe99ab4507a24af4ba7fa5b5278c5ac6 -> Peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Couldn't send request to peer 9204a82012ac492dbc0da9c84c51dabf. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.133:42523: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:26.637123 2862 raft_consensus.cc:482] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3170be3b2bbb4b3bbd8a37ac5f91de89)
I20240501 08:12:26.637326 2862 raft_consensus.cc:504] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 377 OBSOLETE_local: false peers { permanent_uuid: "9204a82012ac492dbc0da9c84c51dabf" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42523 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:26.638259 2862 leader_election.cc:288] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523), 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637), 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 096aec539f1541b4b3049afbdcf36a81 (127.28.1.136:43857)
I20240501 08:12:26.639910 1837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3a5e3d4cc604fcd90a0efeea3946acd" candidate_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" candidate_term: 2 candidate_status { last_received { term: 1 index: 377 } } ignore_live_leader: false dest_uuid: "096aec539f1541b4b3049afbdcf36a81" is_pre_election: true
I20240501 08:12:26.640200 1837 raft_consensus.cc:2454] T b3a5e3d4cc604fcd90a0efeea3946acd P 096aec539f1541b4b3049afbdcf36a81 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe99ab4507a24af4ba7fa5b5278c5ac6 in term 1.
W20240501 08:12:26.640324 32232 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9204a82012ac492dbc0da9c84c51dabf (127.28.1.133:42523): Network error: Client connection negotiation failed: client connection to 127.28.1.133:42523: connect: Connection refused (error 111)
W20240501 08:12:26.640498 32232 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135): Network error: Client connection negotiation failed: client connection to 127.28.1.132:41135: connect: Connection refused (error 111)
W20240501 08:12:26.641000 32231 leader_election.cc:334] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3170be3b2bbb4b3bbd8a37ac5f91de89 (127.28.1.129:39637): Network error: Client connection negotiation failed: client connection to 127.28.1.129:39637: connect: Connection refused (error 111)
I20240501 08:12:26.641117 32231 leader_election.cc:302] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 096aec539f1541b4b3049afbdcf36a81, fe99ab4507a24af4ba7fa5b5278c5ac6; no voters: 3170be3b2bbb4b3bbd8a37ac5f91de89, 9204a82012ac492dbc0da9c84c51dabf, 9fc0640dbc0d4208bfb536949d1bb0c4
I20240501 08:12:26.641371 2862 raft_consensus.cc:2726] T b3a5e3d4cc604fcd90a0efeea3946acd P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:12:26.764123 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 31278
I20240501 08:12:26.804472 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 32213
W20240501 08:12:26.805176 2950 negotiation.cc:337] Failed RPC negotiation. Trace:
0501 08:12:26.545471 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.28.1.134:41579 (local address 127.28.1.136:51717)
0501 08:12:26.545649 (+ 178us) negotiation.cc:107] Waiting for socket to connect
0501 08:12:26.545664 (+ 15us) client_negotiation.cc:174] Beginning negotiation
0501 08:12:26.545784 (+ 120us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0501 08:12:26.804303 (+258519us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.28.1.134:41579: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":110,"mutex_wait_us":6}
W20240501 08:12:26.806277 1772 leader_election.cc:334] T 3c02021c881348748fa99b0df2dfb228 P 096aec539f1541b4b3049afbdcf36a81 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579): Network error: Client connection negotiation failed: client connection to 127.28.1.134:41579: connect: Connection refused (error 111)
I20240501 08:12:26.809573 32340 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:27.039009 2862 raft_consensus.cc:482] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57210dc0a07a4851a43be23a279eb836)
I20240501 08:12:27.039173 2862 raft_consensus.cc:504] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 248 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "1a388864321846929919a37b570e8138" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46291 } } peers { permanent_uuid: "57210dc0a07a4851a43be23a279eb836" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 39451 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } }
I20240501 08:12:27.039984 2862 leader_election.cc:288] T a465d27916734303bd9d1bed2a3dfb14 P fe99ab4507a24af4ba7fa5b5278c5ac6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9fc0640dbc0d4208bfb536949d1bb0c4 (127.28.1.132:41135), 6f37f9ed957342dba1271066c1bbeade (127.28.1.134:41579), 1a388864321846929919a37b570e8138 (127.28.1.130:46291), 57210dc0a07a4851a43be23a279eb836 (127.28.1.131:39451)
I20240501 08:12:27.040657 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 32213
W20240501 08:12:27.078948 1770 connection.cc:535] server connection from 127.28.1.135:52199 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:12:27.079308 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 1755
I20240501 08:12:27.084892 1876 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:27.248009 2949 raft_consensus.cc:482] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 9fc0640dbc0d4208bfb536949d1bb0c4)
I20240501 08:12:27.248236 2949 raft_consensus.cc:504] T 9ff3c78386224499b7022dc9334ddd33 P 096aec539f1541b4b3049afbdcf36a81 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 446 OBSOLETE_local: false peers { permanent_uuid: "9fc0640dbc0d4208bfb536949d1bb0c4" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 41135 } } peers { permanent_uuid: "3170be3b2bbb4b3bbd8a37ac5f91de89" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 39637 } } peers { permanent_uuid: "6f37f9ed957342dba1271066c1bbeade" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 41579 } } peers { permanent_uuid: "fe99ab4507a24af4ba7fa5b5278c5ac6" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 36533 } attrs { promote: false } } peers { permanent_uuid: "096aec539f1541b4b3049afbdcf36a81" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 43857 } attrs { promote: false } }
I20240501 08:12:27.248804 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 1755
I20240501 08:12:27.271209 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 30537
I20240501 08:12:27.272256 30598 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:27.447669 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 30537
2024-05-01T08:12:27Z chronyd exiting
[ OK ] RebalanceParamTest.Rebalance/7 (90010 ms)
[----------] 1 test from RebalanceParamTest (90010 ms total)
[----------] 1 test from TserverAddedDuringRebalancingTest
[ RUN ] TserverAddedDuringRebalancingTest.TserverStarts/1
I20240501 08:12:27.569855 28678 test_util.cc:268] Using random seed: 203006629
I20240501 08:12:27.571451 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:12:27.571522 28678 ts_itest-base.cc:113] --------------
I20240501 08:12:27.571559 28678 ts_itest-base.cc:114] 8 tablet servers
I20240501 08:12:27.571621 28678 ts_itest-base.cc:115] 3 replicas per TS
I20240501 08:12:27.571682 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:12:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:12:27Z Disabled control of system clock
I20240501 08:12:27.615996 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:38185
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:38185
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240501 08:12:27.935549 2978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:27.935869 2978 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:27.935983 2978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:27.943365 2978 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:27.943506 2978 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:12:27.943578 2978 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:27.943634 2978 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:12:27.943683 2978 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:12:27.952989 2978 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:38185
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:38185
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:27.954275 2978 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:27.955952 2978 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:27.965960 2983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:27.966436 2984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:27.966779 2986 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:27.967211 2978 server_base.cc:1034] running on GCE node
I20240501 08:12:27.967931 2978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:27.969316 2978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:27.970503 2978 hybrid_clock.cc:648] HybridClock initialized: now 1714551147970457 us; error 43 us; skew 500 ppm
I20240501 08:12:27.970808 2978 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:27.972929 2978 webserver.cc:460] Webserver started at http://127.28.1.190:44419/ using document root <none> and password file <none>
I20240501 08:12:27.973618 2978 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:27.973724 2978 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:27.974021 2978 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:27.976531 2978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "975c4e11a5ae4beabe81775b2a092a40"
format_stamp: "Formatted at 2024-05-01 08:12:27 on dist-test-slave-jj4n"
I20240501 08:12:27.977226 2978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "975c4e11a5ae4beabe81775b2a092a40"
format_stamp: "Formatted at 2024-05-01 08:12:27 on dist-test-slave-jj4n"
I20240501 08:12:27.982007 2978 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240501 08:12:27.985177 2992 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:27.986410 2978 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:12:27.986580 2978 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "975c4e11a5ae4beabe81775b2a092a40"
format_stamp: "Formatted at 2024-05-01 08:12:27 on dist-test-slave-jj4n"
I20240501 08:12:27.986760 2978 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:28.008352 2978 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:28.009161 2978 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:28.009406 2978 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:28.025491 2978 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:38185
I20240501 08:12:28.025516 3043 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:38185 every 8 connection(s)
I20240501 08:12:28.026724 2978 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:12:28.030165 3044 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:28.031072 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 2978
I20240501 08:12:28.031250 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:12:28.039947 3044 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40: Bootstrap starting.
I20240501 08:12:28.043030 3044 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:28.044001 3044 log.cc:824] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:28.046187 3044 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40: No bootstrap required, opened a new log
I20240501 08:12:28.051735 3044 raft_consensus.cc:348] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } }
I20240501 08:12:28.052065 3044 raft_consensus.cc:374] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:28.052160 3044 raft_consensus.cc:729] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 975c4e11a5ae4beabe81775b2a092a40, State: Initialized, Role: FOLLOWER
I20240501 08:12:28.052717 3044 consensus_queue.cc:260] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } }
I20240501 08:12:28.052896 3044 raft_consensus.cc:388] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:12:28.053020 3044 raft_consensus.cc:482] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:12:28.053151 3044 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:28.055034 3044 raft_consensus.cc:504] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } }
I20240501 08:12:28.055464 3044 leader_election.cc:302] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 975c4e11a5ae4beabe81775b2a092a40; no voters:
I20240501 08:12:28.055984 3044 leader_election.cc:288] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:12:28.056107 3049 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:28.056674 3049 raft_consensus.cc:686] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [term 1 LEADER]: Becoming Leader. State: Replica: 975c4e11a5ae4beabe81775b2a092a40, State: Running, Role: LEADER
I20240501 08:12:28.057163 3049 consensus_queue.cc:237] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } }
I20240501 08:12:28.057782 3044 sys_catalog.cc:564] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:12:28.059574 3050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "975c4e11a5ae4beabe81775b2a092a40" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } } }
I20240501 08:12:28.059924 3050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [sys.catalog]: This master's current role is: LEADER
I20240501 08:12:28.060250 3051 sys_catalog.cc:455] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 975c4e11a5ae4beabe81775b2a092a40. Latest consensus state: current_term: 1 leader_uuid: "975c4e11a5ae4beabe81775b2a092a40" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "975c4e11a5ae4beabe81775b2a092a40" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 38185 } } }
I20240501 08:12:28.060494 3051 sys_catalog.cc:458] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40 [sys.catalog]: This master's current role is: LEADER
I20240501 08:12:28.061810 3057 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:12:28.066677 3057 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:12:28.076066 3057 catalog_manager.cc:1347] Generated new cluster ID: 5d6d48ace9fb486a8ac27ac0d8541818
I20240501 08:12:28.076179 3057 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:12:28.087337 3057 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:12:28.089063 3057 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:12:28.112144 3057 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 975c4e11a5ae4beabe81775b2a092a40: Generated new TSK 0
I20240501 08:12:28.113061 3057 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:12:28.128211 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240501 08:12:28.463093 3068 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:28.463407 3068 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:28.463546 3068 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:28.471122 3068 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:28.471268 3068 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:28.471446 3068 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:12:28.480758 3068 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:28.482081 3068 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:28.483819 3068 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:28.495597 3076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:28.495918 3073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:28.496438 3074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:28.496958 3068 server_base.cc:1034] running on GCE node
I20240501 08:12:28.497593 3068 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:28.498561 3068 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:28.499773 3068 hybrid_clock.cc:648] HybridClock initialized: now 1714551148499715 us; error 41 us; skew 500 ppm
I20240501 08:12:28.500115 3068 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:28.502305 3068 webserver.cc:460] Webserver started at http://127.28.1.129:36011/ using document root <none> and password file <none>
I20240501 08:12:28.503048 3068 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:28.503162 3068 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:28.503476 3068 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:28.506191 3068 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "466a9d70da1b4b1e8579e40447ce0c25"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.506980 3068 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "466a9d70da1b4b1e8579e40447ce0c25"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.512398 3068 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:12:28.515882 3082 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:28.517324 3068 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:12:28.517511 3068 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "466a9d70da1b4b1e8579e40447ce0c25"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.517716 3068 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:28.538652 3068 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:28.539525 3068 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:28.539795 3068 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:28.540782 3068 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:28.542299 3068 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:28.542397 3068 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:28.542512 3068 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:28.542582 3068 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:28.571785 3068 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:42945
I20240501 08:12:28.571844 3194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:42945 every 8 connection(s)
I20240501 08:12:28.573084 3068 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:12:28.574100 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3068
I20240501 08:12:28.574318 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:12:28.577775 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:28.586129 3195 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:28.586503 3195 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:28.587548 3195 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:28.589670 3009 ts_manager.cc:194] Registered new tserver with Master: 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:28.591282 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:49847
W20240501 08:12:28.902205 3199 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:28.902551 3199 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:28.902696 3199 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:28.910193 3199 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:28.910348 3199 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:28.910514 3199 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:12:28.919605 3199 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:28.920954 3199 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:28.922772 3199 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:28.934475 3204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:28.934882 3205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:28.935267 3207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:28.935866 3199 server_base.cc:1034] running on GCE node
I20240501 08:12:28.936517 3199 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:28.937495 3199 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:28.938706 3199 hybrid_clock.cc:648] HybridClock initialized: now 1714551148938655 us; error 47 us; skew 500 ppm
I20240501 08:12:28.939023 3199 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:28.941277 3199 webserver.cc:460] Webserver started at http://127.28.1.130:35587/ using document root <none> and password file <none>
I20240501 08:12:28.941998 3199 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:28.942107 3199 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:28.942425 3199 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:28.945150 3199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.945894 3199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.951100 3199 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240501 08:12:28.954491 3213 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:28.955837 3199 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:28.956035 3199 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
format_stamp: "Formatted at 2024-05-01 08:12:28 on dist-test-slave-jj4n"
I20240501 08:12:28.956233 3199 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:28.971261 3199 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:28.972162 3199 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:28.972442 3199 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:28.973381 3199 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:28.974886 3199 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:28.974979 3199 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:28.975093 3199 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:28.975162 3199 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.004096 3199 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:40531
I20240501 08:12:29.004149 3325 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:40531 every 8 connection(s)
I20240501 08:12:29.005381 3199 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:12:29.006356 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3199
I20240501 08:12:29.006548 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:12:29.009776 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:29.018857 3326 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:29.019208 3326 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:29.020252 3326 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:29.021762 3009 ts_manager.cc:194] Registered new tserver with Master: e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:29.022523 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:46615
W20240501 08:12:29.359576 3329 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:29.359882 3329 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:29.360054 3329 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:29.367660 3329 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:29.367795 3329 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:29.367980 3329 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:12:29.377329 3329 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:29.378659 3329 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:29.380407 3329 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:29.392264 3334 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:29.392630 3337 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:29.393028 3335 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:29.393437 3329 server_base.cc:1034] running on GCE node
I20240501 08:12:29.394045 3329 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:29.394989 3329 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:29.396188 3329 hybrid_clock.cc:648] HybridClock initialized: now 1714551149396135 us; error 42 us; skew 500 ppm
I20240501 08:12:29.396495 3329 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:29.398676 3329 webserver.cc:460] Webserver started at http://127.28.1.131:44645/ using document root <none> and password file <none>
I20240501 08:12:29.399392 3329 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:29.399513 3329 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:29.399832 3329 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:29.402441 3329 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "254a7288f105497eb60d1709036a71a9"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.403190 3329 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "254a7288f105497eb60d1709036a71a9"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.408381 3329 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:12:29.411667 3343 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.413089 3329 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:12:29.413254 3329 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "254a7288f105497eb60d1709036a71a9"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.413456 3329 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:29.430348 3329 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:29.431231 3329 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:29.431509 3329 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:29.432489 3329 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:29.433990 3329 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:29.434085 3329 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.434195 3329 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:29.434262 3329 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.463225 3329 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:44953
I20240501 08:12:29.463281 3455 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:44953 every 8 connection(s)
I20240501 08:12:29.464576 3329 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:12:29.474635 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3329
I20240501 08:12:29.474836 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:12:29.476346 3456 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:29.476621 3456 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:29.477432 3456 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:29.477809 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:29.478807 3009 ts_manager.cc:194] Registered new tserver with Master: 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:29.479605 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:46441
I20240501 08:12:29.594286 3195 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:29.805362 3459 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:29.805655 3459 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:29.805764 3459 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:29.813189 3459 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:29.813315 3459 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:29.813441 3459 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:12:29.822412 3459 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:29.823736 3459 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:29.825525 3459 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:29.837052 3464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:29.837365 3465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:29.837841 3467 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:29.838376 3459 server_base.cc:1034] running on GCE node
I20240501 08:12:29.838963 3459 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:29.839897 3459 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:29.841118 3459 hybrid_clock.cc:648] HybridClock initialized: now 1714551149841069 us; error 39 us; skew 500 ppm
I20240501 08:12:29.841423 3459 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:29.843582 3459 webserver.cc:460] Webserver started at http://127.28.1.132:45093/ using document root <none> and password file <none>
I20240501 08:12:29.844339 3459 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:29.844450 3459 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:29.844756 3459 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:29.847227 3459 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "1771142e14884570b63c687e982902aa"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.847944 3459 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "1771142e14884570b63c687e982902aa"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.852874 3459 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.000s
I20240501 08:12:29.856117 3473 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.857379 3459 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240501 08:12:29.857528 3459 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "1771142e14884570b63c687e982902aa"
format_stamp: "Formatted at 2024-05-01 08:12:29 on dist-test-slave-jj4n"
I20240501 08:12:29.857738 3459 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:29.876462 3459 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:29.877312 3459 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:29.877580 3459 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:29.878525 3459 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:29.880015 3459 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:29.880123 3459 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.880229 3459 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:29.880295 3459 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:29.909739 3459 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:43555
I20240501 08:12:29.909790 3585 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:43555 every 8 connection(s)
I20240501 08:12:29.911100 3459 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:12:29.914155 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3459
I20240501 08:12:29.914322 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:12:29.917259 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.133:0
--local_ip_for_outbound_sockets=127.28.1.133
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:29.924008 3586 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:29.924376 3586 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:29.925395 3586 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:29.926918 3009 ts_manager.cc:194] Registered new tserver with Master: 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:29.927909 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:44753
I20240501 08:12:30.024982 3326 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:30.249645 3589 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:30.249953 3589 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:30.250087 3589 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:30.257467 3589 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:30.257623 3589 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:30.257774 3589 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.133
I20240501 08:12:30.267112 3589 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.28.1.133
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.133
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:30.268486 3589 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:30.270180 3589 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:30.281440 3594 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:30.281891 3595 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:30.282208 3597 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:30.282583 3589 server_base.cc:1034] running on GCE node
I20240501 08:12:30.283146 3589 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:30.284049 3589 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:30.285274 3589 hybrid_clock.cc:648] HybridClock initialized: now 1714551150285214 us; error 51 us; skew 500 ppm
I20240501 08:12:30.285593 3589 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:30.287690 3589 webserver.cc:460] Webserver started at http://127.28.1.133:33551/ using document root <none> and password file <none>
I20240501 08:12:30.288450 3589 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:30.288561 3589 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:30.288868 3589 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:30.291501 3589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "f0af5c767c1b496cb1163870b60b066a"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.292295 3589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "f0af5c767c1b496cb1163870b60b066a"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.297370 3589 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.002s
I20240501 08:12:30.300542 3603 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.301806 3589 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
I20240501 08:12:30.301960 3589 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "f0af5c767c1b496cb1163870b60b066a"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.302152 3589 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:30.315146 3589 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:30.316018 3589 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:30.316288 3589 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:30.317232 3589 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:30.318807 3589 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:30.318905 3589 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.319012 3589 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:30.319078 3589 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.348399 3589 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.133:43351
I20240501 08:12:30.348444 3715 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.133:43351 every 8 connection(s)
I20240501 08:12:30.349687 3589 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240501 08:12:30.355650 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3589
I20240501 08:12:30.355849 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240501 08:12:30.358820 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.134:0
--local_ip_for_outbound_sockets=127.28.1.134
--webserver_interface=127.28.1.134
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:30.362252 3716 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:30.362547 3716 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:30.363394 3716 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:30.364812 3009 ts_manager.cc:194] Registered new tserver with Master: f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351)
I20240501 08:12:30.365783 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.133:50545
I20240501 08:12:30.481979 3456 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:30.686846 3719 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:30.687144 3719 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:30.687264 3719 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:30.697342 3719 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:30.697476 3719 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:30.697625 3719 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.134
I20240501 08:12:30.706606 3719 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.28.1.134
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.134
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:30.707896 3719 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:30.709622 3719 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:30.721241 3727 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:30.721500 3724 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:30.721817 3725 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:30.722378 3719 server_base.cc:1034] running on GCE node
I20240501 08:12:30.722990 3719 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:30.723901 3719 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:30.725114 3719 hybrid_clock.cc:648] HybridClock initialized: now 1714551150725067 us; error 40 us; skew 500 ppm
I20240501 08:12:30.725406 3719 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:30.727537 3719 webserver.cc:460] Webserver started at http://127.28.1.134:39311/ using document root <none> and password file <none>
I20240501 08:12:30.728284 3719 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:30.728407 3719 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:30.728713 3719 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:30.731189 3719 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "8151eaf288354d60b73b3c090dd271fd"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.731904 3719 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "8151eaf288354d60b73b3c090dd271fd"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.736814 3719 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
I20240501 08:12:30.740092 3733 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.741356 3719 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240501 08:12:30.741515 3719 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "8151eaf288354d60b73b3c090dd271fd"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:30.741704 3719 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:30.783206 3719 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:30.784116 3719 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:30.784380 3719 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:30.785352 3719 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:30.786830 3719 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:30.786926 3719 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.787031 3719 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:30.787098 3719 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:30.815881 3719 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.134:44365
I20240501 08:12:30.815950 3845 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.134:44365 every 8 connection(s)
I20240501 08:12:30.817198 3719 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240501 08:12:30.818287 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3719
I20240501 08:12:30.818467 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240501 08:12:30.821847 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.135:0
--local_ip_for_outbound_sockets=127.28.1.135
--webserver_interface=127.28.1.135
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:30.829905 3846 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:30.830219 3846 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:30.831030 3846 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:30.832490 3009 ts_manager.cc:194] Registered new tserver with Master: 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365)
I20240501 08:12:30.833243 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.134:32913
I20240501 08:12:30.930476 3586 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:31.219102 3849 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:31.219399 3849 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:31.219533 3849 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:31.227047 3849 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:31.227252 3849 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:31.227421 3849 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.135
I20240501 08:12:31.236968 3849 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.28.1.135
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.135
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:31.238287 3849 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:31.240100 3849 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:31.251906 3855 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:31.252207 3854 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:31.252561 3857 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:31.253176 3849 server_base.cc:1034] running on GCE node
I20240501 08:12:31.253779 3849 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:31.254716 3849 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:31.255914 3849 hybrid_clock.cc:648] HybridClock initialized: now 1714551151255868 us; error 45 us; skew 500 ppm
I20240501 08:12:31.256265 3849 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:31.258612 3849 webserver.cc:460] Webserver started at http://127.28.1.135:33231/ using document root <none> and password file <none>
I20240501 08:12:31.259378 3849 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:31.259506 3849 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:31.259820 3849 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:31.263012 3849 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "3e6a44464d7748fa8c784ef795b896fb"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.263893 3849 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "3e6a44464d7748fa8c784ef795b896fb"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.269148 3849 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240501 08:12:31.272715 3863 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.274077 3849 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:12:31.274230 3849 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "3e6a44464d7748fa8c784ef795b896fb"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.274420 3849 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:31.302098 3849 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:31.302938 3849 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:31.303205 3849 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:31.304201 3849 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:31.305701 3849 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:31.305796 3849 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.305886 3849 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:31.305939 3849 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.334861 3849 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.135:42375
I20240501 08:12:31.334929 3975 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.135:42375 every 8 connection(s)
I20240501 08:12:31.336217 3849 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240501 08:12:31.340919 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3849
I20240501 08:12:31.341076 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240501 08:12:31.344281 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.136:0
--local_ip_for_outbound_sockets=127.28.1.136
--webserver_interface=127.28.1.136
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:31.349490 3976 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:31.349875 3976 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:31.350931 3976 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:31.352654 3009 ts_manager.cc:194] Registered new tserver with Master: 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:12:31.353627 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.135:38077
I20240501 08:12:31.368167 3716 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:31.675158 3980 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:31.675462 3980 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:31.675608 3980 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:31.683177 3980 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:31.683323 3980 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:31.683486 3980 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.136
I20240501 08:12:31.692662 3980 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.28.1.136
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.136
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:31.693976 3980 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:31.695710 3980 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:31.707382 3985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:31.707724 3986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:31.708179 3988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:31.708725 3980 server_base.cc:1034] running on GCE node
I20240501 08:12:31.709321 3980 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:31.710258 3980 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:31.711454 3980 hybrid_clock.cc:648] HybridClock initialized: now 1714551151711394 us; error 50 us; skew 500 ppm
I20240501 08:12:31.711769 3980 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:31.714128 3980 webserver.cc:460] Webserver started at http://127.28.1.136:37315/ using document root <none> and password file <none>
I20240501 08:12:31.714876 3980 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:31.714988 3980 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:31.715291 3980 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:31.718089 3980 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.718868 3980 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.723913 3980 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
I20240501 08:12:31.727491 3994 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.728866 3980 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
I20240501 08:12:31.729017 3980 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:31.729205 3980 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:31.755832 3980 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:31.756744 3980 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:31.757004 3980 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:31.757966 3980 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:31.759423 3980 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:31.759522 3980 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.759627 3980 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:31.759694 3980 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:31.789160 3980 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.136:42523
I20240501 08:12:31.789228 4106 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.136:42523 every 8 connection(s)
I20240501 08:12:31.790448 3980 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240501 08:12:31.793494 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 3980
I20240501 08:12:31.793665 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240501 08:12:31.802172 4107 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:31.802429 4107 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:31.803171 4107 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:31.804420 3009 ts_manager.cc:194] Registered new tserver with Master: b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523)
I20240501 08:12:31.805155 3009 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.136:48073
I20240501 08:12:31.809572 28678 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240501 08:12:31.825716 28678 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240501 08:12:31.825848 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3589
I20240501 08:12:31.833633 3711 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:31.836202 3846 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
I20240501 08:12:31.913365 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3589
I20240501 08:12:31.926381 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3719
I20240501 08:12:31.934139 3841 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:32.011806 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3719
I20240501 08:12:32.024749 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3849
I20240501 08:12:32.032881 3971 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:32.112457 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3849
I20240501 08:12:32.125322 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3980
I20240501 08:12:32.133668 4102 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:12:32.215058 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3980
I20240501 08:12:35.980085 3009 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:36.023553 3260 tablet_service.cc:1467] Processing CreateTablet for tablet d110b6fb2855454892ab5359ec3210b0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.022169 3261 tablet_service.cc:1467] Processing CreateTablet for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.024880 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d110b6fb2855454892ab5359ec3210b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.026274 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.028430 3130 tablet_service.cc:1467] Processing CreateTablet for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.028950 3128 tablet_service.cc:1467] Processing CreateTablet for tablet 6e2593a1a02d4f61972ba63e7dbc628c (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.029631 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.028215 3129 tablet_service.cc:1467] Processing CreateTablet for tablet d110b6fb2855454892ab5359ec3210b0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.031075 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6e2593a1a02d4f61972ba63e7dbc628c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.031651 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d110b6fb2855454892ab5359ec3210b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.041828 3521 tablet_service.cc:1467] Processing CreateTablet for tablet d110b6fb2855454892ab5359ec3210b0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.042938 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 6e2593a1a02d4f61972ba63e7dbc628c (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.044180 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6e2593a1a02d4f61972ba63e7dbc628c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.045516 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d110b6fb2855454892ab5359ec3210b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.051280 4137 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:36.056977 4137 tablet_bootstrap.cc:654] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.059134 4137 log.cc:824] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:36.058928 3390 tablet_service.cc:1467] Processing CreateTablet for tablet 6e2593a1a02d4f61972ba63e7dbc628c (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.059674 4136 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:36.060250 3391 tablet_service.cc:1467] Processing CreateTablet for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 (DEFAULT_TABLE table=rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:36.061223 4138 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:36.061906 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6e2593a1a02d4f61972ba63e7dbc628c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.062436 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:36.064376 4136 tablet_bootstrap.cc:654] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.065266 4138 tablet_bootstrap.cc:654] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.065613 4136 log.cc:824] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:36.066444 4138 log.cc:824] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:36.067579 4137 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:36.071560 4137 ts_tablet_manager.cc:1367] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.021s user 0.008s sys 0.000s
I20240501 08:12:36.083791 4142 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:36.085731 4137 raft_consensus.cc:348] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.086212 4137 raft_consensus.cc:374] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.086324 4137 raft_consensus.cc:729] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.087056 4137 consensus_queue.cc:260] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.087453 4142 tablet_bootstrap.cc:654] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.089586 4137 ts_tablet_manager.cc:1398] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.018s user 0.011s sys 0.000s
I20240501 08:12:36.090042 4137 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:36.090157 4136 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:36.090425 4138 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:36.090512 4136 ts_tablet_manager.cc:1367] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.031s user 0.010s sys 0.015s
I20240501 08:12:36.090723 4138 ts_tablet_manager.cc:1367] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.030s user 0.005s sys 0.021s
I20240501 08:12:36.091894 4137 tablet_bootstrap.cc:654] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.094810 4142 log.cc:824] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:36.099105 4138 raft_consensus.cc:348] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.099635 4138 raft_consensus.cc:374] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.099750 4138 raft_consensus.cc:729] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.100674 4138 consensus_queue.cc:260] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.101784 4136 raft_consensus.cc:348] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.102272 4136 raft_consensus.cc:374] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.102375 4136 raft_consensus.cc:729] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.103098 4136 consensus_queue.cc:260] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.104810 4142 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:36.105147 4142 ts_tablet_manager.cc:1367] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.022s user 0.007s sys 0.003s
I20240501 08:12:36.110621 4137 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:36.110826 4137 ts_tablet_manager.cc:1367] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.021s user 0.010s sys 0.007s
I20240501 08:12:36.112017 4137 raft_consensus.cc:348] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.112282 4137 raft_consensus.cc:374] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.112380 4137 raft_consensus.cc:729] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.112697 4137 consensus_queue.cc:260] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.114111 4137 ts_tablet_manager.cc:1398] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240501 08:12:36.114634 4138 ts_tablet_manager.cc:1398] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.024s user 0.017s sys 0.005s
I20240501 08:12:36.115063 4138 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:36.116803 4138 tablet_bootstrap.cc:654] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.119956 4142 raft_consensus.cc:348] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.120574 4142 raft_consensus.cc:374] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.120716 4142 raft_consensus.cc:729] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.121960 4136 ts_tablet_manager.cc:1398] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.031s user 0.006s sys 0.019s
I20240501 08:12:36.122119 4138 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:36.122259 4138 ts_tablet_manager.cc:1367] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.003s
I20240501 08:12:36.122517 4136 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:36.123289 4138 raft_consensus.cc:348] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.123500 4138 raft_consensus.cc:374] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.123593 4138 raft_consensus.cc:729] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.123239 4142 consensus_queue.cc:260] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.123805 4138 consensus_queue.cc:260] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.124358 4136 tablet_bootstrap.cc:654] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.124644 4138 ts_tablet_manager.cc:1398] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:36.127902 4142 ts_tablet_manager.cc:1398] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.023s user 0.015s sys 0.000s
I20240501 08:12:36.128338 4136 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:36.128342 4142 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:36.128479 4136 ts_tablet_manager.cc:1367] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.001s
I20240501 08:12:36.129448 4136 raft_consensus.cc:348] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.129649 4136 raft_consensus.cc:374] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.129740 4136 raft_consensus.cc:729] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.129956 4142 tablet_bootstrap.cc:654] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.130031 4136 consensus_queue.cc:260] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.130627 4136 ts_tablet_manager.cc:1398] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:36.130878 4136 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:36.132440 4136 tablet_bootstrap.cc:654] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:36.136719 4143 raft_consensus.cc:482] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:36.136996 4143 raft_consensus.cc:504] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.139075 4143 leader_election.cc:288] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:36.139199 4136 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:36.139371 4136 ts_tablet_manager.cc:1367] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20240501 08:12:36.140281 4142 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:36.140440 4142 ts_tablet_manager.cc:1367] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.005s
I20240501 08:12:36.140900 4136 raft_consensus.cc:348] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.141135 4136 raft_consensus.cc:374] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.141211 4136 raft_consensus.cc:729] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.141451 4142 raft_consensus.cc:348] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.141449 4136 consensus_queue.cc:260] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.141635 4142 raft_consensus.cc:374] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:36.141724 4142 raft_consensus.cc:729] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:36.142120 4136 ts_tablet_manager.cc:1398] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240501 08:12:36.142019 4142 consensus_queue.cc:260] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.142608 4142 ts_tablet_manager.cc:1398] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:36.154882 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d110b6fb2855454892ab5359ec3210b0" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:36.155341 3150 raft_consensus.cc:2454] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:36.155931 3216 leader_election.cc:302] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:36.156339 4143 raft_consensus.cc:2781] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:36.156462 4143 raft_consensus.cc:482] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:36.156556 4143 raft_consensus.cc:3037] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.156559 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d110b6fb2855454892ab5359ec3210b0" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:36.156989 3541 raft_consensus.cc:2454] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:36.158169 4146 raft_consensus.cc:482] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:36.158334 4146 raft_consensus.cc:504] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.159132 4143 raft_consensus.cc:504] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.159811 4143 leader_election.cc:288] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:36.160181 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d110b6fb2855454892ab5359ec3210b0" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:36.160233 4146 leader_election.cc:288] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:36.160373 3150 raft_consensus.cc:3037] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.161476 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d110b6fb2855454892ab5359ec3210b0" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:36.161695 3541 raft_consensus.cc:3037] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.162950 3150 raft_consensus.cc:2454] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:36.165020 3541 raft_consensus.cc:2454] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:36.165436 3217 leader_election.cc:302] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:36.165776 4143 raft_consensus.cc:2781] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
W20240501 08:12:36.166867 3587 tablet.cc:2369] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:12:36.168520 4143 raft_consensus.cc:686] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:36.169049 4143 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:36.169996 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6e2593a1a02d4f61972ba63e7dbc628c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:36.170550 3411 raft_consensus.cc:2454] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:36.171039 3474 leader_election.cc:302] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9; no voters:
I20240501 08:12:36.171415 4146 raft_consensus.cc:2781] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:36.171535 4146 raft_consensus.cc:482] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:36.171624 4146 raft_consensus.cc:3037] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.174299 4146 raft_consensus.cc:504] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.175004 4146 leader_election.cc:288] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:36.174700 3009 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:36.175691 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6e2593a1a02d4f61972ba63e7dbc628c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:36.176002 3149 raft_consensus.cc:3037] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.176153 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6e2593a1a02d4f61972ba63e7dbc628c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:36.176927 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6e2593a1a02d4f61972ba63e7dbc628c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:36.177100 3411 raft_consensus.cc:3037] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.178119 3149 raft_consensus.cc:2454] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:36.179177 3411 raft_consensus.cc:2454] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:36.179625 3476 leader_election.cc:302] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:36.179908 4146 raft_consensus.cc:2781] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:36.180330 4146 raft_consensus.cc:686] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:36.180837 4146 consensus_queue.cc:237] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:36.184767 3009 catalog_manager.cc:5484] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } }
W20240501 08:12:36.220564 3457 tablet.cc:2369] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:12:36.221647 4147 raft_consensus.cc:482] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:36.221822 4147 raft_consensus.cc:504] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.223465 4147 leader_election.cc:288] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:36.231304 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e62d72ab5f9b460aaa7fe7bb7ecd7b08" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:36.231623 3411 raft_consensus.cc:2454] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:36.231513 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e62d72ab5f9b460aaa7fe7bb7ecd7b08" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:36.231946 3281 raft_consensus.cc:2454] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:36.232170 3083 leader_election.cc:302] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:36.232580 4147 raft_consensus.cc:2781] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:36.232740 4147 raft_consensus.cc:482] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:36.232841 4147 raft_consensus.cc:3037] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.234702 4147 raft_consensus.cc:504] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.235294 4147 leader_election.cc:288] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:36.235765 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e62d72ab5f9b460aaa7fe7bb7ecd7b08" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:36.235841 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e62d72ab5f9b460aaa7fe7bb7ecd7b08" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:36.236011 3411 raft_consensus.cc:3037] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.236047 3281 raft_consensus.cc:3037] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:36.237816 3411 raft_consensus.cc:2454] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:36.238031 3281 raft_consensus.cc:2454] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:36.238238 3083 leader_election.cc:302] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:36.238526 4147 raft_consensus.cc:2781] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:36.238912 4147 raft_consensus.cc:686] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Becoming Leader. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Running, Role: LEADER
I20240501 08:12:36.239362 4147 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:36.243692 3009 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: term changed from 0 to 1, leader changed from <none> to 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
W20240501 08:12:36.261581 3327 tablet.cc:2369] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:12:36.329926 3196 tablet.cc:2369] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:12:36.599620 4165 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:36.599936 4165 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:36.607399 4165 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:12:36.619472 4147 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:36.630975 4147 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:36.636103 4143 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:36.641408 4143 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
I20240501 08:12:36.759452 4146 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:36.769419 3150 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:36.770337 4146 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:36.817750 4175 mvcc.cc:205] Tried to move back new op lower bound from 7022801538115485696 to 7022801535720726528. Current Snapshot: MvccSnapshot[applied={T|T < 7022801538115485696}]
INSERT report
rows total: 2000
time total: 218.958 ms
time per row: 0.109479 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:12:37.844053 3009 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:37.852922 3260 tablet_service.cc:1467] Processing CreateTablet for tablet 7667242ae1c548ba982e74e190f2af27 (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.852922 3390 tablet_service.cc:1467] Processing CreateTablet for tablet 7667242ae1c548ba982e74e190f2af27 (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.853101 3521 tablet_service.cc:1467] Processing CreateTablet for tablet 7667242ae1c548ba982e74e190f2af27 (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.853364 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7667242ae1c548ba982e74e190f2af27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.853364 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7667242ae1c548ba982e74e190f2af27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.853506 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7667242ae1c548ba982e74e190f2af27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.857141 3261 tablet_service.cc:1467] Processing CreateTablet for tablet 1c3672fbf92d4b54a91554f88382a5ae (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.857125 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 1c3672fbf92d4b54a91554f88382a5ae (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.857296 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 885de7b67f474f7bb54badfae193906d (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.857492 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c3672fbf92d4b54a91554f88382a5ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.857503 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c3672fbf92d4b54a91554f88382a5ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.857712 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 885de7b67f474f7bb54badfae193906d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.860348 3259 tablet_service.cc:1467] Processing CreateTablet for tablet 885de7b67f474f7bb54badfae193906d (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.860713 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 885de7b67f474f7bb54badfae193906d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.861160 3389 tablet_service.cc:1467] Processing CreateTablet for tablet 885de7b67f474f7bb54badfae193906d (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.861207 3391 tablet_service.cc:1467] Processing CreateTablet for tablet 1c3672fbf92d4b54a91554f88382a5ae (DEFAULT_TABLE table=rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:37.861615 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 885de7b67f474f7bb54badfae193906d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.862402 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c3672fbf92d4b54a91554f88382a5ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:37.872956 4235 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:37.873829 4234 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:37.874819 4235 tablet_bootstrap.cc:654] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.875649 4234 tablet_bootstrap.cc:654] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.879707 4236 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:37.881577 4236 tablet_bootstrap.cc:654] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.881896 4237 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:37.882081 4235 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:37.882221 4235 ts_tablet_manager.cc:1367] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
I20240501 08:12:37.882947 4235 raft_consensus.cc:348] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.883116 4235 raft_consensus.cc:374] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.883194 4235 raft_consensus.cc:729] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.883471 4237 tablet_bootstrap.cc:654] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.883409 4235 consensus_queue.cc:260] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.883590 4236 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:37.883730 4236 ts_tablet_manager.cc:1367] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:37.884006 4235 ts_tablet_manager.cc:1398] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:37.884276 4235 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:37.885017 4236 raft_consensus.cc:348] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.885232 4236 raft_consensus.cc:374] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.885313 4236 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.885560 4236 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.885721 4235 tablet_bootstrap.cc:654] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.887374 4234 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:37.887532 4234 ts_tablet_manager.cc:1367] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.014s user 0.001s sys 0.007s
I20240501 08:12:37.887925 4235 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:37.888092 4235 ts_tablet_manager.cc:1367] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.001s
I20240501 08:12:37.888290 4234 raft_consensus.cc:348] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.888481 4234 raft_consensus.cc:374] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.888561 4234 raft_consensus.cc:729] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.888770 4235 raft_consensus.cc:348] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.888803 4234 consensus_queue.cc:260] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.888967 4235 raft_consensus.cc:374] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.889029 4235 raft_consensus.cc:729] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.889299 4234 ts_tablet_manager.cc:1398] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:37.889575 4234 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:37.892903 4234 tablet_bootstrap.cc:654] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.889271 4235 consensus_queue.cc:260] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.894969 4235 ts_tablet_manager.cc:1398] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240501 08:12:37.895623 4236 ts_tablet_manager.cc:1398] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.012s user 0.004s sys 0.007s
I20240501 08:12:37.896497 4237 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:37.896657 4237 ts_tablet_manager.cc:1367] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.015s user 0.001s sys 0.012s
I20240501 08:12:37.896869 4236 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:37.897387 4237 raft_consensus.cc:348] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.897598 4237 raft_consensus.cc:374] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.897691 4237 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.897953 4237 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.898593 4237 ts_tablet_manager.cc:1398] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:37.899646 4236 tablet_bootstrap.cc:654] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.900128 4234 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:37.900935 4234 ts_tablet_manager.cc:1367] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.004s
I20240501 08:12:37.901557 4234 raft_consensus.cc:348] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.901737 4234 raft_consensus.cc:374] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.901809 4234 raft_consensus.cc:729] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.902027 4234 consensus_queue.cc:260] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.902491 4234 ts_tablet_manager.cc:1398] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:37.902753 4234 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:37.902858 4236 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:37.902978 4236 ts_tablet_manager.cc:1367] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.001s
I20240501 08:12:37.903952 4234 tablet_bootstrap.cc:654] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.904013 4236 raft_consensus.cc:348] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.904234 4236 raft_consensus.cc:374] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.904322 4236 raft_consensus.cc:729] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.904578 4236 consensus_queue.cc:260] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.905817 4234 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:37.905957 4234 ts_tablet_manager.cc:1367] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:37.906394 4236 ts_tablet_manager.cc:1398] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:37.906541 4234 raft_consensus.cc:348] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.906708 4236 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:37.906726 4234 raft_consensus.cc:374] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.906811 4234 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.907011 4234 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:37.907629 4234 ts_tablet_manager.cc:1398] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:37.908191 4236 tablet_bootstrap.cc:654] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:37.910043 4236 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:37.910169 4236 ts_tablet_manager.cc:1367] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:37.910710 4236 raft_consensus.cc:348] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.910866 4236 raft_consensus.cc:374] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:37.910948 4236 raft_consensus.cc:729] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:37.911164 4236 consensus_queue.cc:260] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.911613 4236 ts_tablet_manager.cc:1398] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:37.938097 4242 raft_consensus.cc:482] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:37.938303 4242 raft_consensus.cc:504] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.940009 4242 leader_election.cc:288] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:37.947578 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c3672fbf92d4b54a91554f88382a5ae" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:37.947824 3281 raft_consensus.cc:2454] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:37.948386 3347 leader_election.cc:302] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:37.948793 4242 raft_consensus.cc:2781] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:37.948918 4242 raft_consensus.cc:482] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:37.949016 4242 raft_consensus.cc:3037] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:37.949635 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c3672fbf92d4b54a91554f88382a5ae" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:37.949862 3541 raft_consensus.cc:2454] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:37.951205 4242 raft_consensus.cc:504] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.951761 4242 leader_election.cc:288] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:37.952188 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c3672fbf92d4b54a91554f88382a5ae" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:37.952261 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c3672fbf92d4b54a91554f88382a5ae" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:37.952378 3281 raft_consensus.cc:3037] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:37.952466 3541 raft_consensus.cc:3037] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:37.954119 3281 raft_consensus.cc:2454] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:37.954226 3541 raft_consensus.cc:2454] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:37.954492 3347 leader_election.cc:302] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:37.954788 4242 raft_consensus.cc:2781] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:37.955173 4242 raft_consensus.cc:686] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:37.955627 4242 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:37.958916 3008 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:38.077486 4242 raft_consensus.cc:482] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:38.077745 4242 raft_consensus.cc:504] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:38.078446 4242 leader_election.cc:288] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:38.079070 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:38.079320 3281 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:38.079708 3347 leader_election.cc:302] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:38.080047 4242 raft_consensus.cc:2781] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:38.080139 4242 raft_consensus.cc:482] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:38.080246 4242 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.082160 4242 raft_consensus.cc:504] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:38.082743 4242 leader_election.cc:288] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:38.083336 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:38.083541 3281 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.084612 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:38.084656 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:38.084856 3150 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:38.085718 3281 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:38.086035 3347 leader_election.cc:302] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters: 466a9d70da1b4b1e8579e40447ce0c25
I20240501 08:12:38.086270 4242 raft_consensus.cc:2781] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:38.086426 4242 raft_consensus.cc:686] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:38.086684 4242 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:38.089061 3008 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:38.286657 4183 raft_consensus.cc:482] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:38.286871 4183 raft_consensus.cc:504] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:38.287642 4183 leader_election.cc:288] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:38.288328 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7667242ae1c548ba982e74e190f2af27" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:38.288604 3541 raft_consensus.cc:2454] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:38.289018 3217 leader_election.cc:302] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:38.289306 4183 raft_consensus.cc:2781] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:38.289422 4183 raft_consensus.cc:482] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:38.289526 4183 raft_consensus.cc:3037] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.291550 4183 raft_consensus.cc:504] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:38.292141 4183 leader_election.cc:288] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:38.292603 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7667242ae1c548ba982e74e190f2af27" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:38.292788 3541 raft_consensus.cc:3037] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.294199 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7667242ae1c548ba982e74e190f2af27" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:38.294312 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7667242ae1c548ba982e74e190f2af27" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:38.294440 3410 raft_consensus.cc:2454] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:38.294937 3541 raft_consensus.cc:2454] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:38.295251 3217 leader_election.cc:302] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters: 254a7288f105497eb60d1709036a71a9
I20240501 08:12:38.295517 4183 raft_consensus.cc:2781] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:38.295985 4183 raft_consensus.cc:686] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:38.296288 4183 consensus_queue.cc:237] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:38.298674 3008 catalog_manager.cc:5484] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:38.339264 4242 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:38.350265 4252 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:38.591413 3150 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.594889 4252 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:38.599099 4267 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:38.658567 4260 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:38.658883 4260 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:38.666433 4260 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240501 08:12:38.765023 3410 raft_consensus.cc:3037] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:38.767710 4183 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:38.772816 4258 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 208.664 ms
time per row: 0.104332 ms
I20240501 08:12:39.010982 3003 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:39.020120 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 4cb5c6a514354708bff63dd5e51c7420 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.020582 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cb5c6a514354708bff63dd5e51c7420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.020737 3389 tablet_service.cc:1467] Processing CreateTablet for tablet 0bd2ea6b1e714d619d7dc00d20067a59 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.021114 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bd2ea6b1e714d619d7dc00d20067a59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.024700 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 4cb5c6a514354708bff63dd5e51c7420 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.025118 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cb5c6a514354708bff63dd5e51c7420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.026896 3259 tablet_service.cc:1467] Processing CreateTablet for tablet d749fd4842294f8fba36f287b7072c9d (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.027292 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d749fd4842294f8fba36f287b7072c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.028409 3260 tablet_service.cc:1467] Processing CreateTablet for tablet 4cb5c6a514354708bff63dd5e51c7420 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.028947 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cb5c6a514354708bff63dd5e51c7420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.032425 3128 tablet_service.cc:1467] Processing CreateTablet for tablet d749fd4842294f8fba36f287b7072c9d (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.032842 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d749fd4842294f8fba36f287b7072c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.036392 4322 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:39.037019 3521 tablet_service.cc:1467] Processing CreateTablet for tablet 0bd2ea6b1e714d619d7dc00d20067a59 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.037379 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bd2ea6b1e714d619d7dc00d20067a59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.038152 4322 tablet_bootstrap.cc:654] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.040338 3129 tablet_service.cc:1467] Processing CreateTablet for tablet 0bd2ea6b1e714d619d7dc00d20067a59 (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.040719 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bd2ea6b1e714d619d7dc00d20067a59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.044641 3519 tablet_service.cc:1467] Processing CreateTablet for tablet d749fd4842294f8fba36f287b7072c9d (DEFAULT_TABLE table=rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:39.045099 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d749fd4842294f8fba36f287b7072c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:39.046200 4322 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:39.046358 4322 ts_tablet_manager.cc:1367] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20240501 08:12:39.047196 4322 raft_consensus.cc:348] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.047415 4322 raft_consensus.cc:374] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.047502 4322 raft_consensus.cc:729] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.047756 4322 consensus_queue.cc:260] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.048338 4322 ts_tablet_manager.cc:1398] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:39.048631 4322 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:39.050279 4322 tablet_bootstrap.cc:654] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.067605 4326 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:39.069869 4326 tablet_bootstrap.cc:654] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.073458 4326 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:39.074131 4326 ts_tablet_manager.cc:1367] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240501 08:12:39.074519 4325 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:39.075004 4326 raft_consensus.cc:348] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.075241 4326 raft_consensus.cc:374] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.075359 4326 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.075659 4326 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.076380 4321 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:39.077955 4321 tablet_bootstrap.cc:654] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.079180 4326 ts_tablet_manager.cc:1398] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240501 08:12:39.079517 4326 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:39.080973 4326 tablet_bootstrap.cc:654] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.082422 4321 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:39.082571 4321 ts_tablet_manager.cc:1367] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240501 08:12:39.083362 4321 raft_consensus.cc:348] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.083573 4321 raft_consensus.cc:374] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.083669 4321 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.083907 4321 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.084498 4321 ts_tablet_manager.cc:1398] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:39.084772 4321 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:39.086386 4325 tablet_bootstrap.cc:654] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.086457 4326 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:39.086608 4326 ts_tablet_manager.cc:1367] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:12:39.087354 4326 raft_consensus.cc:348] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.087564 4326 raft_consensus.cc:374] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.087642 4326 raft_consensus.cc:729] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.087889 4326 consensus_queue.cc:260] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.088414 4326 ts_tablet_manager.cc:1398] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:39.088701 4326 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:39.090137 4326 tablet_bootstrap.cc:654] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.092026 4321 tablet_bootstrap.cc:654] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.093878 4322 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:39.094035 4322 ts_tablet_manager.cc:1367] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.045s user 0.020s sys 0.023s
I20240501 08:12:39.094621 4322 raft_consensus.cc:348] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.094789 4322 raft_consensus.cc:374] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.094861 4322 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.095073 4322 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.095551 4322 ts_tablet_manager.cc:1398] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:39.097301 4183 raft_consensus.cc:482] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:39.097436 4183 raft_consensus.cc:504] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.097993 4183 leader_election.cc:288] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:39.098352 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:39.098518 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:39.098591 3150 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:39.098752 3541 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:39.099018 3216 leader_election.cc:302] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:39.099298 4183 raft_consensus.cc:2781] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:39.099403 4183 raft_consensus.cc:482] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:39.099481 4183 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.100633 4325 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:39.100795 4325 ts_tablet_manager.cc:1367] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.026s user 0.003s sys 0.002s
I20240501 08:12:39.101528 4183 raft_consensus.cc:504] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.101575 4325 raft_consensus.cc:348] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.101783 4325 raft_consensus.cc:374] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.101874 4325 raft_consensus.cc:729] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.102079 4183 leader_election.cc:288] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:39.102450 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:39.102452 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:39.102634 3541 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.102640 3150 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.102524 4325 consensus_queue.cc:260] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.103161 4325 ts_tablet_manager.cc:1398] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:39.105291 3150 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:39.105719 3216 leader_election.cc:302] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:39.105934 4321 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:39.106083 4321 ts_tablet_manager.cc:1367] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.006s
I20240501 08:12:39.105995 4183 raft_consensus.cc:2781] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:39.106382 4183 raft_consensus.cc:686] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:39.106686 4183 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.106817 4321 raft_consensus.cc:348] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.107019 4321 raft_consensus.cc:374] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.107085 4321 raft_consensus.cc:729] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.107309 4321 consensus_queue.cc:260] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.107843 4321 ts_tablet_manager.cc:1398] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:39.108137 4321 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:39.108747 3541 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:39.109607 4321 tablet_bootstrap.cc:654] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:39.112783 4326 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:39.112922 4326 ts_tablet_manager.cc:1367] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.024s user 0.005s sys 0.005s
I20240501 08:12:39.113337 4321 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:39.113166 3003 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:39.113600 4326 raft_consensus.cc:348] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.113806 4326 raft_consensus.cc:374] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.113906 4326 raft_consensus.cc:729] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.113963 4321 ts_tablet_manager.cc:1367] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
I20240501 08:12:39.114140 4326 consensus_queue.cc:260] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.114643 4326 ts_tablet_manager.cc:1398] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:39.114634 4321 raft_consensus.cc:348] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.114823 4321 raft_consensus.cc:374] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:39.114909 4321 raft_consensus.cc:729] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:39.115108 4321 consensus_queue.cc:260] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.116372 4321 ts_tablet_manager.cc:1398] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:39.243628 4229 raft_consensus.cc:482] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:39.243826 4229 raft_consensus.cc:504] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.244515 4229 leader_election.cc:288] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:39.245149 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bd2ea6b1e714d619d7dc00d20067a59" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:39.245149 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bd2ea6b1e714d619d7dc00d20067a59" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:39.245399 3150 raft_consensus.cc:2454] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:39.245404 3410 raft_consensus.cc:2454] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:39.245795 3476 leader_election.cc:302] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:39.246040 4229 raft_consensus.cc:2781] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:39.246150 4229 raft_consensus.cc:482] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:39.246212 4229 raft_consensus.cc:3037] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.248054 4229 raft_consensus.cc:504] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.248591 4229 leader_election.cc:288] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:39.248982 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bd2ea6b1e714d619d7dc00d20067a59" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:39.249100 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bd2ea6b1e714d619d7dc00d20067a59" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:39.249200 3410 raft_consensus.cc:3037] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.249321 3150 raft_consensus.cc:3037] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.251940 3410 raft_consensus.cc:2454] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:39.251940 3150 raft_consensus.cc:2454] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:39.252382 3476 leader_election.cc:302] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:39.252652 4229 raft_consensus.cc:2781] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:39.252982 4229 raft_consensus.cc:686] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:39.253252 4229 consensus_queue.cc:237] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:39.255791 3009 catalog_manager.cc:5484] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:39.335213 4183 raft_consensus.cc:482] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:39.335390 4183 raft_consensus.cc:504] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.336069 4183 leader_election.cc:288] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:39.336532 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d749fd4842294f8fba36f287b7072c9d" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:39.336563 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d749fd4842294f8fba36f287b7072c9d" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:39.336726 3150 raft_consensus.cc:2454] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:39.336814 3541 raft_consensus.cc:2454] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:39.337127 3216 leader_election.cc:302] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:39.337432 4183 raft_consensus.cc:2781] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:39.337527 4183 raft_consensus.cc:482] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:39.337630 4183 raft_consensus.cc:3037] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.339479 4183 raft_consensus.cc:504] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.340044 4183 leader_election.cc:288] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:39.340514 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d749fd4842294f8fba36f287b7072c9d" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:39.340675 3541 raft_consensus.cc:3037] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.342028 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d749fd4842294f8fba36f287b7072c9d" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:39.342207 3150 raft_consensus.cc:3037] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:39.342597 3541 raft_consensus.cc:2454] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:39.342963 3217 leader_election.cc:302] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:39.343217 4183 raft_consensus.cc:2781] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:39.343372 4183 raft_consensus.cc:686] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:39.343631 4183 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:39.344309 3150 raft_consensus.cc:2454] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:39.346191 3009 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:39.578202 4183 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:39.594424 4340 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:39.666368 4335 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:39.670786 4335 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:39.744649 4338 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:39.745014 4338 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:39.752344 4338 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240501 08:12:39.871101 3540 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:39.872171 4299 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:39.875109 3149 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:39.876125 4345 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:39.965989 4341 mvcc.cc:205] Tried to move back new op lower bound from 7022801550844116992 to 7022801548673818624. Current Snapshot: MvccSnapshot[applied={T|T < 7022801550822465536}]
I20240501 08:12:40.034844 4383 mvcc.cc:205] Tried to move back new op lower bound from 7022801550822465536 to 7022801548673818624. Current Snapshot: MvccSnapshot[applied={T|T < 7022801550822465536}]
INSERT report
rows total: 2000
time total: 241.657 ms
time per row: 0.120828 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:12:40.993597 3001 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:41.001928 3389 tablet_service.cc:1467] Processing CreateTablet for tablet a3fce8868ca74a028492122fc2762001 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.002214 3259 tablet_service.cc:1467] Processing CreateTablet for tablet a3fce8868ca74a028492122fc2762001 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.002405 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a3fce8868ca74a028492122fc2762001. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.002619 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a3fce8868ca74a028492122fc2762001. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.002605 3130 tablet_service.cc:1467] Processing CreateTablet for tablet a3fce8868ca74a028492122fc2762001 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.002990 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a3fce8868ca74a028492122fc2762001. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.003504 3128 tablet_service.cc:1467] Processing CreateTablet for tablet d3db152c64e74ee3a788769dba03b7a6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.003844 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d3db152c64e74ee3a788769dba03b7a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.005543 3260 tablet_service.cc:1467] Processing CreateTablet for tablet 460948db7dfd4d5589d478ddd5c6c10c (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.005867 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 460948db7dfd4d5589d478ddd5c6c10c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.008837 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 460948db7dfd4d5589d478ddd5c6c10c (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.009254 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 460948db7dfd4d5589d478ddd5c6c10c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.011870 3519 tablet_service.cc:1467] Processing CreateTablet for tablet d3db152c64e74ee3a788769dba03b7a6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.012221 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d3db152c64e74ee3a788769dba03b7a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.015342 3391 tablet_service.cc:1467] Processing CreateTablet for tablet 460948db7dfd4d5589d478ddd5c6c10c (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.015264 3390 tablet_service.cc:1467] Processing CreateTablet for tablet d3db152c64e74ee3a788769dba03b7a6 (DEFAULT_TABLE table=rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:41.017488 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d3db152c64e74ee3a788769dba03b7a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.018085 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 460948db7dfd4d5589d478ddd5c6c10c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:41.034996 4407 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:41.036641 4407 tablet_bootstrap.cc:654] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.044970 4406 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:41.045553 4408 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:41.046448 4406 tablet_bootstrap.cc:654] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.047111 4408 tablet_bootstrap.cc:654] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.048230 4409 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:41.048343 4406 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:41.048488 4406 ts_tablet_manager.cc:1367] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:41.049695 4409 tablet_bootstrap.cc:654] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.050622 4406 raft_consensus.cc:348] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.050844 4406 raft_consensus.cc:374] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.050927 4406 raft_consensus.cc:729] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.051823 4406 consensus_queue.cc:260] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.052341 4406 ts_tablet_manager.cc:1398] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:12:41.053113 4406 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:41.053782 4407 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:41.053989 4407 ts_tablet_manager.cc:1367] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.019s user 0.005s sys 0.003s
I20240501 08:12:41.054530 4406 tablet_bootstrap.cc:654] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.054836 4407 raft_consensus.cc:348] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.055107 4407 raft_consensus.cc:374] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.055243 4407 raft_consensus.cc:729] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.055536 4407 consensus_queue.cc:260] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.056200 4407 ts_tablet_manager.cc:1398] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:41.057246 4407 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:41.062145 4407 tablet_bootstrap.cc:654] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.062363 4406 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:41.062531 4406 ts_tablet_manager.cc:1367] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.009s user 0.005s sys 0.003s
I20240501 08:12:41.063444 4408 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:41.063329 4406 raft_consensus.cc:348] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.063539 4406 raft_consensus.cc:374] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.063612 4406 raft_consensus.cc:729] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.063745 4408 ts_tablet_manager.cc:1367] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.018s user 0.001s sys 0.009s
I20240501 08:12:41.063870 4406 consensus_queue.cc:260] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.064457 4406 ts_tablet_manager.cc:1398] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:41.064750 4406 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:41.064778 4408 raft_consensus.cc:348] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.065091 4408 raft_consensus.cc:374] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.065217 4408 raft_consensus.cc:729] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.065511 4408 consensus_queue.cc:260] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.066190 4408 ts_tablet_manager.cc:1398] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:41.066485 4408 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:41.067608 4406 tablet_bootstrap.cc:654] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.067993 4408 tablet_bootstrap.cc:654] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.070699 4406 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:41.070874 4406 ts_tablet_manager.cc:1367] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240501 08:12:41.071578 4406 raft_consensus.cc:348] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.071808 4406 raft_consensus.cc:374] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.071915 4406 raft_consensus.cc:729] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.072177 4406 consensus_queue.cc:260] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.072810 4406 ts_tablet_manager.cc:1398] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:41.075258 4407 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:41.075269 4409 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:41.075443 4407 ts_tablet_manager.cc:1367] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.000s
I20240501 08:12:41.075469 4409 ts_tablet_manager.cc:1367] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.027s user 0.010s sys 0.010s
I20240501 08:12:41.076185 4407 raft_consensus.cc:348] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.076262 4409 raft_consensus.cc:348] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.076407 4407 raft_consensus.cc:374] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.076439 4409 raft_consensus.cc:374] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.076499 4407 raft_consensus.cc:729] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.076529 4409 raft_consensus.cc:729] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.076723 4409 consensus_queue.cc:260] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.076742 4407 consensus_queue.cc:260] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.077261 4407 ts_tablet_manager.cc:1398] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:41.077831 4409 ts_tablet_manager.cc:1398] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:41.078106 4409 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:41.078229 4408 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:41.078379 4408 ts_tablet_manager.cc:1367] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.005s
I20240501 08:12:41.079068 4408 raft_consensus.cc:348] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.079263 4408 raft_consensus.cc:374] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.079331 4408 raft_consensus.cc:729] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.079648 4409 tablet_bootstrap.cc:654] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:41.079592 4408 consensus_queue.cc:260] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.080755 4408 ts_tablet_manager.cc:1398] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:41.082377 4409 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:41.082522 4409 ts_tablet_manager.cc:1367] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240501 08:12:41.083142 4409 raft_consensus.cc:348] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.083307 4409 raft_consensus.cc:374] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:41.083379 4409 raft_consensus.cc:729] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:41.083592 4409 consensus_queue.cc:260] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.084141 4409 ts_tablet_manager.cc:1398] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:41.088086 4242 raft_consensus.cc:482] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:41.088243 4242 raft_consensus.cc:504] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.088856 4242 leader_election.cc:288] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:41.089273 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d3db152c64e74ee3a788769dba03b7a6" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:41.089311 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d3db152c64e74ee3a788769dba03b7a6" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:41.089507 3148 raft_consensus.cc:2454] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:41.089509 3541 raft_consensus.cc:2454] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:41.089931 3346 leader_election.cc:302] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:41.090201 4242 raft_consensus.cc:2781] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:41.090322 4242 raft_consensus.cc:482] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:41.090430 4242 raft_consensus.cc:3037] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.092263 4242 raft_consensus.cc:504] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.092751 4242 leader_election.cc:288] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:41.093171 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d3db152c64e74ee3a788769dba03b7a6" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:41.093156 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d3db152c64e74ee3a788769dba03b7a6" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:41.093338 3148 raft_consensus.cc:3037] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.093356 3541 raft_consensus.cc:3037] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.095916 3148 raft_consensus.cc:2454] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:41.095902 3541 raft_consensus.cc:2454] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:41.096323 3346 leader_election.cc:302] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:41.096568 4242 raft_consensus.cc:2781] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:41.096915 4242 raft_consensus.cc:686] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:41.097160 4242 consensus_queue.cc:237] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.099586 3003 catalog_manager.cc:5484] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:41.128077 4179 raft_consensus.cc:482] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:41.128297 4179 raft_consensus.cc:504] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.129091 4179 leader_election.cc:288] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:41.129417 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a3fce8868ca74a028492122fc2762001" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:41.129604 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a3fce8868ca74a028492122fc2762001" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:41.129665 3411 raft_consensus.cc:2454] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:41.129842 3281 raft_consensus.cc:2454] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:41.130115 3083 leader_election.cc:302] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:41.130497 4179 raft_consensus.cc:2781] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:41.130641 4179 raft_consensus.cc:482] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:41.130733 4179 raft_consensus.cc:3037] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.133083 4179 raft_consensus.cc:504] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.133576 4179 leader_election.cc:288] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:41.133997 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a3fce8868ca74a028492122fc2762001" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:41.134069 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a3fce8868ca74a028492122fc2762001" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:41.134238 3411 raft_consensus.cc:3037] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.134244 3281 raft_consensus.cc:3037] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.136929 3281 raft_consensus.cc:2454] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:41.136929 3411 raft_consensus.cc:2454] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:41.137295 3083 leader_election.cc:302] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:41.137590 4179 raft_consensus.cc:2781] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:41.137921 4179 raft_consensus.cc:686] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Becoming Leader. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Running, Role: LEADER
I20240501 08:12:41.138231 4179 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } }
I20240501 08:12:41.140776 3003 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: term changed from 0 to 1, leader changed from <none> to 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:41.202746 4345 raft_consensus.cc:482] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:41.202988 4345 raft_consensus.cc:504] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.203596 4345 leader_election.cc:288] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:41.204206 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "460948db7dfd4d5589d478ddd5c6c10c" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:41.204411 3541 raft_consensus.cc:2454] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:41.204697 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "460948db7dfd4d5589d478ddd5c6c10c" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:41.204803 3217 leader_election.cc:302] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:41.204892 3411 raft_consensus.cc:2454] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:41.205101 4345 raft_consensus.cc:2781] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:41.205189 4345 raft_consensus.cc:482] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:41.205299 4345 raft_consensus.cc:3037] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.207124 4345 raft_consensus.cc:504] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.207621 4345 leader_election.cc:288] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:41.208037 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "460948db7dfd4d5589d478ddd5c6c10c" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:41.208081 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "460948db7dfd4d5589d478ddd5c6c10c" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:41.208186 3541 raft_consensus.cc:3037] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.208262 3411 raft_consensus.cc:3037] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:41.209901 3541 raft_consensus.cc:2454] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:41.210076 3411 raft_consensus.cc:2454] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:41.210250 3217 leader_election.cc:302] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:41.210497 4345 raft_consensus.cc:2781] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:41.210665 4345 raft_consensus.cc:686] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:41.210906 4345 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:41.213438 3003 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:41.480679 4242 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:41.486824 4242 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:41.543915 4424 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:41.544342 4424 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:41.551622 4424 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240501 08:12:41.622685 4345 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:41.638137 4401 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:41.698453 4179 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:41.779651 4474 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 226.472 ms
time per row: 0.113236 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:12:42.820822 3001 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:42.829877 3128 tablet_service.cc:1467] Processing CreateTablet for tablet a68f4cc374ad4656b1de788033ece98c (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.830364 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a68f4cc374ad4656b1de788033ece98c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.830876 3391 tablet_service.cc:1467] Processing CreateTablet for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.831274 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f0c4a54ade9c4e96bb575bfbfa5d0206. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.831279 3129 tablet_service.cc:1467] Processing CreateTablet for tablet 313e1bed969b4a4ba124116ba5aed741 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.831456 3519 tablet_service.cc:1467] Processing CreateTablet for tablet a68f4cc374ad4656b1de788033ece98c (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.831655 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313e1bed969b4a4ba124116ba5aed741. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.831854 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a68f4cc374ad4656b1de788033ece98c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.833576 3130 tablet_service.cc:1467] Processing CreateTablet for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.833925 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f0c4a54ade9c4e96bb575bfbfa5d0206. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.835997 3260 tablet_service.cc:1467] Processing CreateTablet for tablet a68f4cc374ad4656b1de788033ece98c (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.836390 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a68f4cc374ad4656b1de788033ece98c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.836897 3390 tablet_service.cc:1467] Processing CreateTablet for tablet 313e1bed969b4a4ba124116ba5aed741 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.837265 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313e1bed969b4a4ba124116ba5aed741. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.840638 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 313e1bed969b4a4ba124116ba5aed741 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.841009 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313e1bed969b4a4ba124116ba5aed741. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.836063 3259 tablet_service.cc:1467] Processing CreateTablet for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 (DEFAULT_TABLE table=rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:42.842715 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f0c4a54ade9c4e96bb575bfbfa5d0206. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:42.846450 4497 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:42.848227 4497 tablet_bootstrap.cc:654] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.853160 4498 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:42.853307 4497 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:42.853451 4497 ts_tablet_manager.cc:1367] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
I20240501 08:12:42.853542 4499 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:42.854192 4497 raft_consensus.cc:348] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.854410 4497 raft_consensus.cc:374] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.854517 4497 raft_consensus.cc:729] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.854749 4497 consensus_queue.cc:260] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.855289 4499 tablet_bootstrap.cc:654] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.857666 4498 tablet_bootstrap.cc:654] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.858170 4497 ts_tablet_manager.cc:1398] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:12:42.859807 4497 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:42.862547 4497 tablet_bootstrap.cc:654] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.866729 4498 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:42.866883 4498 ts_tablet_manager.cc:1367] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.007s
I20240501 08:12:42.867635 4498 raft_consensus.cc:348] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.867847 4498 raft_consensus.cc:374] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.867942 4498 raft_consensus.cc:729] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.868202 4498 consensus_queue.cc:260] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.868664 4496 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:42.869107 4497 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:42.869261 4497 ts_tablet_manager.cc:1367] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.000s
I20240501 08:12:42.869419 4498 ts_tablet_manager.cc:1398] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:42.869657 4498 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:42.870018 4497 raft_consensus.cc:348] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.870234 4497 raft_consensus.cc:374] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.870299 4497 raft_consensus.cc:729] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.870590 4497 consensus_queue.cc:260] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.870792 4499 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:42.870939 4499 ts_tablet_manager.cc:1367] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.005s
I20240501 08:12:42.871220 4497 ts_tablet_manager.cc:1398] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:42.871708 4498 tablet_bootstrap.cc:654] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.871660 4499 raft_consensus.cc:348] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.871843 4499 raft_consensus.cc:374] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.871929 4499 raft_consensus.cc:729] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.872170 4499 consensus_queue.cc:260] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.872778 4499 ts_tablet_manager.cc:1398] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:42.873016 4499 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:42.874217 4499 tablet_bootstrap.cc:654] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.875908 4496 tablet_bootstrap.cc:654] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.878136 4499 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:42.878278 4499 ts_tablet_manager.cc:1367] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:12:42.878832 4499 raft_consensus.cc:348] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.879024 4499 raft_consensus.cc:374] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.879102 4499 raft_consensus.cc:729] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.879302 4499 consensus_queue.cc:260] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.879582 4498 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:42.879719 4499 ts_tablet_manager.cc:1398] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:42.879737 4498 ts_tablet_manager.cc:1367] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.008s
I20240501 08:12:42.880435 4498 raft_consensus.cc:348] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.880597 4498 raft_consensus.cc:374] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.880589 4496 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:42.880641 4498 raft_consensus.cc:729] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.880728 4496 ts_tablet_manager.cc:1367] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.002s
I20240501 08:12:42.880823 4498 consensus_queue.cc:260] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.881709 4498 ts_tablet_manager.cc:1398] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:42.881435 4496 raft_consensus.cc:348] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.881953 4496 raft_consensus.cc:374] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.881992 4498 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:42.882054 4496 raft_consensus.cc:729] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.882294 4496 consensus_queue.cc:260] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.882825 4496 ts_tablet_manager.cc:1398] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:42.883486 4496 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:42.885038 4496 tablet_bootstrap.cc:654] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.885325 4498 tablet_bootstrap.cc:654] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:42.887028 4496 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:42.887359 4496 ts_tablet_manager.cc:1367] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240501 08:12:42.887607 4498 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:42.887748 4498 ts_tablet_manager.cc:1367] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.001s
I20240501 08:12:42.888034 4496 raft_consensus.cc:348] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.888231 4496 raft_consensus.cc:374] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.888304 4496 raft_consensus.cc:729] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.888386 4498 raft_consensus.cc:348] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.888556 4498 raft_consensus.cc:374] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:42.888640 4498 raft_consensus.cc:729] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:42.888541 4496 consensus_queue.cc:260] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.888758 4345 raft_consensus.cc:482] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:42.888859 4498 consensus_queue.cc:260] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.889048 4496 ts_tablet_manager.cc:1398] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:42.888952 4345 raft_consensus.cc:504] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.889284 4498 ts_tablet_manager.cc:1398] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240501 08:12:42.889570 4345 leader_election.cc:288] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:42.889983 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f0c4a54ade9c4e96bb575bfbfa5d0206" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:42.890041 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f0c4a54ade9c4e96bb575bfbfa5d0206" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:42.890162 3148 raft_consensus.cc:2454] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:42.890251 3411 raft_consensus.cc:2454] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:42.890646 3216 leader_election.cc:302] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:42.891124 4345 raft_consensus.cc:2781] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:42.891224 4345 raft_consensus.cc:482] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:42.891323 4345 raft_consensus.cc:3037] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.893316 4345 raft_consensus.cc:504] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.893848 4345 leader_election.cc:288] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:42.894191 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f0c4a54ade9c4e96bb575bfbfa5d0206" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:42.894243 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f0c4a54ade9c4e96bb575bfbfa5d0206" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:42.894364 3148 raft_consensus.cc:3037] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.894410 3411 raft_consensus.cc:3037] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.896111 3148 raft_consensus.cc:2454] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:42.896147 3411 raft_consensus.cc:2454] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:42.896471 3216 leader_election.cc:302] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:42.896699 4345 raft_consensus.cc:2781] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:42.897025 4345 raft_consensus.cc:686] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:42.897284 4345 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.899755 3001 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:42.913205 4483 raft_consensus.cc:482] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:42.913421 4483 raft_consensus.cc:504] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.914388 4483 leader_election.cc:288] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:42.915048 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313e1bed969b4a4ba124116ba5aed741" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:42.915292 3411 raft_consensus.cc:2454] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:42.915724 3083 leader_election.cc:302] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:42.916083 4483 raft_consensus.cc:2781] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:42.916211 4483 raft_consensus.cc:482] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:42.916303 4483 raft_consensus.cc:3037] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.918736 4483 raft_consensus.cc:504] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.919312 4483 leader_election.cc:288] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:42.919831 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313e1bed969b4a4ba124116ba5aed741" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:42.920112 3411 raft_consensus.cc:3037] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.921234 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313e1bed969b4a4ba124116ba5aed741" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:42.921293 3540 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313e1bed969b4a4ba124116ba5aed741" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:42.921507 3541 raft_consensus.cc:2454] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:42.922415 3411 raft_consensus.cc:2454] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:42.922744 3083 leader_election.cc:302] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters: 1771142e14884570b63c687e982902aa
I20240501 08:12:42.923031 4483 raft_consensus.cc:2781] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:42.923377 4483 raft_consensus.cc:686] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Becoming Leader. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Running, Role: LEADER
I20240501 08:12:42.923633 4483 consensus_queue.cc:237] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:42.926419 3001 catalog_manager.cc:5484] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: term changed from 0 to 1, leader changed from <none> to 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:42.978627 4229 raft_consensus.cc:482] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:42.978825 4229 raft_consensus.cc:504] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.979466 4229 leader_election.cc:288] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:42.980103 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a68f4cc374ad4656b1de788033ece98c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:42.980347 3148 raft_consensus.cc:2454] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:42.980753 3476 leader_election.cc:302] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:42.981082 4229 raft_consensus.cc:2781] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:42.981199 4229 raft_consensus.cc:482] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:42.981316 4229 raft_consensus.cc:3037] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.983397 4229 raft_consensus.cc:504] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.984117 4229 leader_election.cc:288] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:42.984475 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a68f4cc374ad4656b1de788033ece98c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:42.984689 3148 raft_consensus.cc:3037] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:42.986464 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a68f4cc374ad4656b1de788033ece98c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:42.986493 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a68f4cc374ad4656b1de788033ece98c" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:42.986709 3281 raft_consensus.cc:2454] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:42.986711 3148 raft_consensus.cc:2454] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:42.987082 3476 leader_election.cc:302] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:42.987323 4229 raft_consensus.cc:2781] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:42.987663 4229 raft_consensus.cc:686] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:42.987923 4229 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:42.990842 3001 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } }
W20240501 08:12:43.353238 4519 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:43.353570 4519 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:43.360956 4519 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
I20240501 08:12:43.399659 4509 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:43.405472 4509 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:43.410879 4229 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:43.419888 3280 raft_consensus.cc:3037] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.424825 4517 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:43.432642 4483 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:43.438907 3540 raft_consensus.cc:3037] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.441869 4513 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 189.864 ms
time per row: 0.0949321 ms
I20240501 08:12:43.722394 3001 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:43.730537 3259 tablet_service.cc:1467] Processing CreateTablet for tablet f75a6080c937423f8ffb9f75b96da670 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.730801 3519 tablet_service.cc:1467] Processing CreateTablet for tablet f75a6080c937423f8ffb9f75b96da670 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.730988 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75a6080c937423f8ffb9f75b96da670. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.731230 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75a6080c937423f8ffb9f75b96da670. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.731599 3390 tablet_service.cc:1467] Processing CreateTablet for tablet 7fde726fbe8d48f685a4580ff6ae4033 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.731993 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fde726fbe8d48f685a4580ff6ae4033. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.732627 3391 tablet_service.cc:1467] Processing CreateTablet for tablet f75a6080c937423f8ffb9f75b96da670 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.733000 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75a6080c937423f8ffb9f75b96da670. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.737661 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 0612f1f8e4ca4a29971151bf52571a33 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.738091 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0612f1f8e4ca4a29971151bf52571a33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.741114 3260 tablet_service.cc:1467] Processing CreateTablet for tablet 7fde726fbe8d48f685a4580ff6ae4033 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.741151 3521 tablet_service.cc:1467] Processing CreateTablet for tablet 0612f1f8e4ca4a29971151bf52571a33 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.741485 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fde726fbe8d48f685a4580ff6ae4033. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.741557 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0612f1f8e4ca4a29971151bf52571a33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.744339 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 7fde726fbe8d48f685a4580ff6ae4033 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.744699 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fde726fbe8d48f685a4580ff6ae4033. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.748450 3261 tablet_service.cc:1467] Processing CreateTablet for tablet 0612f1f8e4ca4a29971151bf52571a33 (DEFAULT_TABLE table=rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:43.748909 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0612f1f8e4ca4a29971151bf52571a33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:43.752063 4584 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:43.754364 4583 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:43.755052 4582 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:43.755999 4583 tablet_bootstrap.cc:654] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.756208 4584 tablet_bootstrap.cc:654] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.756799 4582 tablet_bootstrap.cc:654] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.761988 4582 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:43.762260 4582 ts_tablet_manager.cc:1367] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240501 08:12:43.763155 4582 raft_consensus.cc:348] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.763495 4582 raft_consensus.cc:374] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.763640 4582 raft_consensus.cc:729] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.763988 4582 consensus_queue.cc:260] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.764662 4582 ts_tablet_manager.cc:1398] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:43.765022 4582 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:43.765638 4581 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:43.766523 4582 tablet_bootstrap.cc:654] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.767911 4581 tablet_bootstrap.cc:654] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.770812 4584 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:43.770835 4582 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:43.770967 4584 ts_tablet_manager.cc:1367] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.019s user 0.005s sys 0.005s
I20240501 08:12:43.770980 4582 ts_tablet_manager.cc:1367] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
I20240501 08:12:43.771054 4583 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:43.771198 4583 ts_tablet_manager.cc:1367] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.017s user 0.001s sys 0.012s
I20240501 08:12:43.771656 4582 raft_consensus.cc:348] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.771732 4584 raft_consensus.cc:348] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.771768 4583 raft_consensus.cc:348] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.771862 4582 raft_consensus.cc:374] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.771907 4584 raft_consensus.cc:374] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.771937 4582 raft_consensus.cc:729] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.771942 4583 raft_consensus.cc:374] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.772027 4584 raft_consensus.cc:729] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.772045 4583 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.772183 4582 consensus_queue.cc:260] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.772280 4583 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.772255 4584 consensus_queue.cc:260] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.772756 4583 ts_tablet_manager.cc:1398] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.001s user 0.000s sys 0.002s
I20240501 08:12:43.773087 4583 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:43.773622 4581 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:43.773759 4581 ts_tablet_manager.cc:1367] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.003s
I20240501 08:12:43.774477 4583 tablet_bootstrap.cc:654] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.774529 4581 raft_consensus.cc:348] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.774721 4581 raft_consensus.cc:374] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.774806 4581 raft_consensus.cc:729] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.774976 4584 ts_tablet_manager.cc:1398] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:12:43.775028 4581 consensus_queue.cc:260] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.775514 4581 ts_tablet_manager.cc:1398] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:43.776013 4582 ts_tablet_manager.cc:1398] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:12:43.776190 4581 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:43.776290 4582 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:43.777438 4581 tablet_bootstrap.cc:654] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.777712 4582 tablet_bootstrap.cc:654] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.778910 4583 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:43.779124 4583 ts_tablet_manager.cc:1367] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
I20240501 08:12:43.779860 4583 raft_consensus.cc:348] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.780164 4583 raft_consensus.cc:374] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.780274 4583 raft_consensus.cc:729] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.780552 4583 consensus_queue.cc:260] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.781610 4583 ts_tablet_manager.cc:1398] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:43.783425 4581 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:43.783573 4581 ts_tablet_manager.cc:1367] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.004s
I20240501 08:12:43.784307 4581 raft_consensus.cc:348] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.784487 4581 raft_consensus.cc:374] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.784560 4581 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.784775 4581 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.785317 4581 ts_tablet_manager.cc:1398] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:43.785601 4581 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:43.786688 4581 tablet_bootstrap.cc:654] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:43.788537 4582 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:43.788690 4582 ts_tablet_manager.cc:1367] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.012s user 0.005s sys 0.007s
I20240501 08:12:43.788919 4581 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:43.789060 4581 ts_tablet_manager.cc:1367] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:43.789449 4582 raft_consensus.cc:348] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.789664 4582 raft_consensus.cc:374] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.789582 4581 raft_consensus.cc:348] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.789752 4581 raft_consensus.cc:374] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:43.789767 4582 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.789842 4581 raft_consensus.cc:729] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:43.789992 4582 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.790069 4581 consensus_queue.cc:260] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.790516 4582 ts_tablet_manager.cc:1398] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:43.790683 4581 ts_tablet_manager.cc:1398] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:43.873175 4536 raft_consensus.cc:482] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:43.873354 4536 raft_consensus.cc:504] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.874032 4536 leader_election.cc:288] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:43.874648 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75a6080c937423f8ffb9f75b96da670" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:43.874648 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75a6080c937423f8ffb9f75b96da670" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:43.874908 3541 raft_consensus.cc:2454] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:43.874908 3411 raft_consensus.cc:2454] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:43.875388 3214 leader_election.cc:302] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:43.875687 4536 raft_consensus.cc:2781] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:43.875816 4536 raft_consensus.cc:482] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:43.875901 4536 raft_consensus.cc:3037] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.878104 4536 raft_consensus.cc:504] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.878612 4536 leader_election.cc:288] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:43.879030 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75a6080c937423f8ffb9f75b96da670" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:43.879045 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75a6080c937423f8ffb9f75b96da670" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:43.879228 3541 raft_consensus.cc:3037] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.879281 3411 raft_consensus.cc:3037] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.881314 3411 raft_consensus.cc:2454] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:43.881350 3541 raft_consensus.cc:2454] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:43.881713 3214 leader_election.cc:302] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:43.882033 4536 raft_consensus.cc:2781] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:43.882174 4536 raft_consensus.cc:686] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:43.882414 4536 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.885285 3006 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:43.907855 4517 raft_consensus.cc:482] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:43.908092 4517 raft_consensus.cc:504] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.908746 4517 leader_election.cc:288] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:43.909291 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0612f1f8e4ca4a29971151bf52571a33" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:43.909284 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0612f1f8e4ca4a29971151bf52571a33" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:43.909520 3149 raft_consensus.cc:2454] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:43.909533 3281 raft_consensus.cc:2454] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:43.909909 3477 leader_election.cc:302] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:43.910215 4517 raft_consensus.cc:2781] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:43.910341 4517 raft_consensus.cc:482] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:43.910455 4517 raft_consensus.cc:3037] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.912364 4517 raft_consensus.cc:504] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.912945 4517 leader_election.cc:288] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:43.913317 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0612f1f8e4ca4a29971151bf52571a33" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:43.913354 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0612f1f8e4ca4a29971151bf52571a33" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:43.913524 3149 raft_consensus.cc:3037] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.913512 3281 raft_consensus.cc:3037] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.915313 3149 raft_consensus.cc:2454] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:43.915663 3476 leader_election.cc:302] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:43.915910 4517 raft_consensus.cc:2781] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:43.916090 4517 raft_consensus.cc:686] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:43.916363 4517 consensus_queue.cc:237] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:43.919198 3006 catalog_manager.cc:5484] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:43.937852 3281 raft_consensus.cc:2454] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:43.957731 4536 raft_consensus.cc:482] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:43.957948 4536 raft_consensus.cc:504] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.958532 4536 leader_election.cc:288] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:43.959013 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:43.959085 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:43.959275 3541 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:43.959283 3411 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:43.959712 3217 leader_election.cc:302] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:43.960053 4536 raft_consensus.cc:2781] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:43.960140 4536 raft_consensus.cc:482] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:43.960242 4536 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.962213 4536 raft_consensus.cc:504] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.962796 4536 leader_election.cc:288] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:43.963178 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:43.963236 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:43.963361 3541 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.963388 3411 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:43.965169 3411 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:43.965538 3214 leader_election.cc:302] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:43.965780 4536 raft_consensus.cc:2781] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:43.965941 4536 raft_consensus.cc:686] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:43.966198 4536 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:43.968695 3006 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:43.990051 3541 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
W20240501 08:12:44.320200 4597 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:44.320505 4597 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:44.327916 4597 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
I20240501 08:12:44.358028 4517 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:44.365511 4536 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:44.373172 4517 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:44.374161 4546 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:44.465487 4546 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:44.476045 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:12:44.477362 4536 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 182.961 ms
time per row: 0.0914803 ms
I20240501 08:12:44.591224 4622 mvcc.cc:205] Tried to move back new op lower bound from 7022801569678659584 to 7022801567607443456. Current Snapshot: MvccSnapshot[applied={T|T < 7022801569678659584}]
I20240501 08:12:44.685802 3006 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:44.694005 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 4adf393d330f45ba95fb50c850f20348 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.694466 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4adf393d330f45ba95fb50c850f20348. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.694628 3390 tablet_service.cc:1467] Processing CreateTablet for tablet 4adf393d330f45ba95fb50c850f20348 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.695020 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4adf393d330f45ba95fb50c850f20348. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.695341 3391 tablet_service.cc:1467] Processing CreateTablet for tablet 901e7e9545d94ee2904a29c5675ccc75 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.695701 3391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 901e7e9545d94ee2904a29c5675ccc75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.700233 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 901e7e9545d94ee2904a29c5675ccc75 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.700317 3389 tablet_service.cc:1467] Processing CreateTablet for tablet c927478b05c14f47b736faf52b4db9aa (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.700313 3261 tablet_service.cc:1467] Processing CreateTablet for tablet c927478b05c14f47b736faf52b4db9aa (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.700619 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 901e7e9545d94ee2904a29c5675ccc75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.700656 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c927478b05c14f47b736faf52b4db9aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.700652 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c927478b05c14f47b736faf52b4db9aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.704298 3259 tablet_service.cc:1467] Processing CreateTablet for tablet 4adf393d330f45ba95fb50c850f20348 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.704651 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4adf393d330f45ba95fb50c850f20348. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.710673 3129 tablet_service.cc:1467] Processing CreateTablet for tablet 901e7e9545d94ee2904a29c5675ccc75 (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.711097 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 901e7e9545d94ee2904a29c5675ccc75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.716043 3128 tablet_service.cc:1467] Processing CreateTablet for tablet c927478b05c14f47b736faf52b4db9aa (DEFAULT_TABLE table=rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:44.716454 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c927478b05c14f47b736faf52b4db9aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:44.716693 4663 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:44.718369 4663 tablet_bootstrap.cc:654] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.723654 4663 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:44.723846 4664 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:44.724256 4663 ts_tablet_manager.cc:1367] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240501 08:12:44.725036 4663 raft_consensus.cc:348] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.725365 4663 raft_consensus.cc:374] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.725507 4663 raft_consensus.cc:729] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.725618 4664 tablet_bootstrap.cc:654] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.725808 4663 consensus_queue.cc:260] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.726429 4663 ts_tablet_manager.cc:1398] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:44.726850 4663 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:44.728794 4663 tablet_bootstrap.cc:654] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.732656 4662 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:44.733160 4665 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:44.733999 4664 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:44.734030 4663 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:44.734145 4664 ts_tablet_manager.cc:1367] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.005s
I20240501 08:12:44.734169 4663 ts_tablet_manager.cc:1367] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.004s
I20240501 08:12:44.734776 4664 raft_consensus.cc:348] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.734829 4663 raft_consensus.cc:348] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.734935 4664 raft_consensus.cc:374] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.735018 4664 raft_consensus.cc:729] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.735080 4663 raft_consensus.cc:374] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.735153 4663 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.735323 4665 tablet_bootstrap.cc:654] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.735245 4664 consensus_queue.cc:260] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.735391 4663 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.735816 4664 ts_tablet_manager.cc:1398] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:12:44.735900 4663 ts_tablet_manager.cc:1398] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:44.736225 4663 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:44.737640 4663 tablet_bootstrap.cc:654] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.738174 4662 tablet_bootstrap.cc:654] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.742389 4662 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:44.742514 4665 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:44.742638 4662 ts_tablet_manager.cc:1367] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.004s
I20240501 08:12:44.742689 4665 ts_tablet_manager.cc:1367] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.005s
I20240501 08:12:44.742992 4663 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:44.743158 4663 ts_tablet_manager.cc:1367] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
I20240501 08:12:44.743484 4665 raft_consensus.cc:348] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.743739 4665 raft_consensus.cc:374] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.743873 4665 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.743858 4663 raft_consensus.cc:348] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.744139 4663 raft_consensus.cc:374] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.744274 4663 raft_consensus.cc:729] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.744181 4665 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.744282 4662 raft_consensus.cc:348] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.744475 4662 raft_consensus.cc:374] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.744575 4662 raft_consensus.cc:729] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.744511 4663 consensus_queue.cc:260] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.744791 4662 consensus_queue.cc:260] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.745188 4665 ts_tablet_manager.cc:1398] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:44.745381 4662 ts_tablet_manager.cc:1398] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240501 08:12:44.745550 4665 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:44.745646 4662 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:44.745668 4663 ts_tablet_manager.cc:1398] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:44.747694 4662 tablet_bootstrap.cc:654] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.748679 4665 tablet_bootstrap.cc:654] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.751456 4665 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:44.751606 4665 ts_tablet_manager.cc:1367] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
I20240501 08:12:44.751873 4662 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:44.752058 4662 ts_tablet_manager.cc:1367] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.004s
I20240501 08:12:44.752398 4665 raft_consensus.cc:348] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.752573 4665 raft_consensus.cc:374] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.752645 4665 raft_consensus.cc:729] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.752749 4662 raft_consensus.cc:348] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.752918 4662 raft_consensus.cc:374] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.752866 4665 consensus_queue.cc:260] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.752985 4662 raft_consensus.cc:729] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.753211 4662 consensus_queue.cc:260] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:44.753710 4662 ts_tablet_manager.cc:1398] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:44.753983 4665 ts_tablet_manager.cc:1398] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:44.754009 4662 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:44.755633 4662 tablet_bootstrap.cc:654] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:44.757372 4662 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:44.757544 4662 ts_tablet_manager.cc:1367] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240501 08:12:44.758296 4662 raft_consensus.cc:348] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.758487 4662 raft_consensus.cc:374] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:44.758589 4662 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:44.758791 4662 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.759239 4662 ts_tablet_manager.cc:1398] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:44.812167 4639 raft_consensus.cc:482] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:44.812347 4639 raft_consensus.cc:504] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.813077 4639 leader_election.cc:288] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:44.813509 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:44.813542 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:44.813755 3149 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:44.813773 3281 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:44.814203 3347 leader_election.cc:302] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:44.814465 4639 raft_consensus.cc:2781] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:44.814571 4639 raft_consensus.cc:482] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:44.814648 4639 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.816581 4639 raft_consensus.cc:504] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.817086 4639 leader_election.cc:288] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:44.817551 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:44.817552 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:44.817710 3149 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.817744 3281 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.819517 3281 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:44.819526 3149 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:44.819887 3346 leader_election.cc:302] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:44.820169 4639 raft_consensus.cc:2781] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:44.820305 4639 raft_consensus.cc:686] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:44.820540 4639 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:44.823413 3009 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:44.916692 4639 raft_consensus.cc:482] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:44.916915 4639 raft_consensus.cc:504] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.917587 4639 leader_election.cc:288] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:44.918116 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4adf393d330f45ba95fb50c850f20348" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:44.918206 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4adf393d330f45ba95fb50c850f20348" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:44.918386 3149 raft_consensus.cc:2454] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:44.918422 3281 raft_consensus.cc:2454] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:44.918865 3346 leader_election.cc:302] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:44.919170 4639 raft_consensus.cc:2781] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:44.919252 4639 raft_consensus.cc:482] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:44.919340 4639 raft_consensus.cc:3037] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.921205 4639 raft_consensus.cc:504] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.921777 4639 leader_election.cc:288] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:44.922154 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4adf393d330f45ba95fb50c850f20348" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:44.922188 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4adf393d330f45ba95fb50c850f20348" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:44.922333 3149 raft_consensus.cc:3037] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.922338 3281 raft_consensus.cc:3037] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:44.924168 3281 raft_consensus.cc:2454] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:44.924202 3149 raft_consensus.cc:2454] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:44.924523 3347 leader_election.cc:302] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:44.924774 4639 raft_consensus.cc:2781] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:44.924923 4639 raft_consensus.cc:686] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:44.925257 4639 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:44.928083 3009 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:45.088723 4639 raft_consensus.cc:482] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:45.088929 4639 raft_consensus.cc:504] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:45.089543 4639 leader_election.cc:288] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:45.090119 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:45.090138 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:45.090355 3149 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:45.090358 3539 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 0.
I20240501 08:12:45.090786 3346 leader_election.cc:302] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:45.091050 4639 raft_consensus.cc:2781] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:45.091185 4639 raft_consensus.cc:482] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:45.091264 4639 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.093261 4639 raft_consensus.cc:504] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:45.093750 4639 leader_election.cc:288] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:45.094199 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:45.094209 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "254a7288f105497eb60d1709036a71a9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:45.094372 3149 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.094383 3539 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.096112 3149 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:45.096122 3539 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 254a7288f105497eb60d1709036a71a9 in term 1.
I20240501 08:12:45.096479 3346 leader_election.cc:302] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:45.096735 4639 raft_consensus.cc:2781] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:45.096853 4639 raft_consensus.cc:686] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Leader. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:45.097163 4639 consensus_queue.cc:237] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:45.100091 3009 catalog_manager.cc:5484] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 254a7288f105497eb60d1709036a71a9 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:45.248365 4639 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:45.254916 4639 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:45.434638 4639 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:45.440980 4419 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:45.513247 4419 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:45.517273 4419 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:45.518260 4678 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:45.518672 4678 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:45.528178 4678 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
INSERT report
rows total: 2000
time total: 194.277 ms
time per row: 0.0971385 ms
I20240501 08:12:45.865190 3003 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:45.873883 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 6da0f7473ef248f2ad4f5621f3fbd25e (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.874347 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6da0f7473ef248f2ad4f5621f3fbd25e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.874619 3519 tablet_service.cc:1467] Processing CreateTablet for tablet fdeb6b683c7f4097ba3054ab8d4fb203 (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.874737 3520 tablet_service.cc:1467] Processing CreateTablet for tablet 6da0f7473ef248f2ad4f5621f3fbd25e (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.874943 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fdeb6b683c7f4097ba3054ab8d4fb203. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.875205 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6da0f7473ef248f2ad4f5621f3fbd25e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.875619 3390 tablet_service.cc:1467] Processing CreateTablet for tablet fdeb6b683c7f4097ba3054ab8d4fb203 (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.875742 3261 tablet_service.cc:1467] Processing CreateTablet for tablet 6da0f7473ef248f2ad4f5621f3fbd25e (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.876163 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6da0f7473ef248f2ad4f5621f3fbd25e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.876588 3389 tablet_service.cc:1467] Processing CreateTablet for tablet e21419bf69e746baa7a96fbf5eee384d (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.877403 3521 tablet_service.cc:1467] Processing CreateTablet for tablet e21419bf69e746baa7a96fbf5eee384d (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.877748 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e21419bf69e746baa7a96fbf5eee384d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.878978 3129 tablet_service.cc:1467] Processing CreateTablet for tablet fdeb6b683c7f4097ba3054ab8d4fb203 (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.879362 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fdeb6b683c7f4097ba3054ab8d4fb203. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.880157 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fdeb6b683c7f4097ba3054ab8d4fb203. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.880363 3259 tablet_service.cc:1467] Processing CreateTablet for tablet e21419bf69e746baa7a96fbf5eee384d (DEFAULT_TABLE table=rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:45.880712 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e21419bf69e746baa7a96fbf5eee384d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.884230 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e21419bf69e746baa7a96fbf5eee384d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:45.892056 4742 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:45.892093 4740 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:45.893784 4740 tablet_bootstrap.cc:654] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.893785 4742 tablet_bootstrap.cc:654] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.901557 4745 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:45.902942 4741 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:45.904374 4745 tablet_bootstrap.cc:654] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.904664 4741 tablet_bootstrap.cc:654] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.907158 4740 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:45.907214 4742 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:45.907313 4740 ts_tablet_manager.cc:1367] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.015s user 0.010s sys 0.003s
I20240501 08:12:45.907436 4742 ts_tablet_manager.cc:1367] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.011s
I20240501 08:12:45.908054 4740 raft_consensus.cc:348] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.908236 4740 raft_consensus.cc:374] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.908319 4740 raft_consensus.cc:729] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.908349 4742 raft_consensus.cc:348] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.908586 4742 raft_consensus.cc:374] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.908563 4740 consensus_queue.cc:260] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.908707 4742 raft_consensus.cc:729] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.909003 4742 consensus_queue.cc:260] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.909240 4741 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:45.909412 4741 ts_tablet_manager.cc:1367] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.003s
I20240501 08:12:45.909680 4742 ts_tablet_manager.cc:1398] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:45.909961 4742 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:45.910254 4741 raft_consensus.cc:348] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.910478 4741 raft_consensus.cc:374] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.910584 4741 raft_consensus.cc:729] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.910821 4741 consensus_queue.cc:260] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.911245 4745 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:45.911404 4745 ts_tablet_manager.cc:1367] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20240501 08:12:45.911427 4741 ts_tablet_manager.cc:1398] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:12:45.912046 4742 tablet_bootstrap.cc:654] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.912528 4741 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:45.912694 4745 raft_consensus.cc:348] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.912890 4745 raft_consensus.cc:374] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.912992 4745 raft_consensus.cc:729] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.914341 4741 tablet_bootstrap.cc:654] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.915421 4740 ts_tablet_manager.cc:1398] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240501 08:12:45.915800 4745 consensus_queue.cc:260] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.916599 4745 ts_tablet_manager.cc:1398] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:12:45.916939 4745 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:45.918874 4745 tablet_bootstrap.cc:654] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.921010 4740 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:45.924067 4740 tablet_bootstrap.cc:654] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.926286 4741 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:45.926584 4741 ts_tablet_manager.cc:1367] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.003s
I20240501 08:12:45.927505 4741 raft_consensus.cc:348] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.927839 4741 raft_consensus.cc:374] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.927958 4741 raft_consensus.cc:729] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.928365 4741 consensus_queue.cc:260] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.928840 4742 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:45.929137 4742 ts_tablet_manager.cc:1367] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.019s user 0.007s sys 0.011s
I20240501 08:12:45.930022 4741 ts_tablet_manager.cc:1398] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:45.930044 4742 raft_consensus.cc:348] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.930356 4742 raft_consensus.cc:374] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.930460 4742 raft_consensus.cc:729] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.930783 4742 consensus_queue.cc:260] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.931414 4745 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:45.931506 4740 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:45.931638 4745 ts_tablet_manager.cc:1367] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.010s
I20240501 08:12:45.931713 4740 ts_tablet_manager.cc:1367] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.005s
I20240501 08:12:45.932475 4742 ts_tablet_manager.cc:1398] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:12:45.932535 4740 raft_consensus.cc:348] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.932808 4742 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:45.932802 4740 raft_consensus.cc:374] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.932878 4740 raft_consensus.cc:729] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.933200 4740 consensus_queue.cc:260] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.933260 4745 raft_consensus.cc:348] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.933539 4745 raft_consensus.cc:374] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.933665 4745 raft_consensus.cc:729] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.933956 4745 consensus_queue.cc:260] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.934185 4740 ts_tablet_manager.cc:1398] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:45.934396 4742 tablet_bootstrap.cc:654] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:45.934535 4745 ts_tablet_manager.cc:1398] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:45.939908 4742 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:45.940196 4742 ts_tablet_manager.cc:1367] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.005s
I20240501 08:12:45.940976 4742 raft_consensus.cc:348] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.941138 4742 raft_consensus.cc:374] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:45.941218 4742 raft_consensus.cc:729] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:45.941403 4742 consensus_queue.cc:260] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.941833 4742 ts_tablet_manager.cc:1398] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:12:45.972198 4517 raft_consensus.cc:482] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:45.972386 4517 raft_consensus.cc:504] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.973034 4517 leader_election.cc:288] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:45.973588 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e21419bf69e746baa7a96fbf5eee384d" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:45.973584 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e21419bf69e746baa7a96fbf5eee384d" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:45.973809 3411 raft_consensus.cc:2454] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:45.973805 3281 raft_consensus.cc:2454] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:45.974257 3474 leader_election.cc:302] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9; no voters:
I20240501 08:12:45.974510 4517 raft_consensus.cc:2781] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:45.974627 4517 raft_consensus.cc:482] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:45.974694 4517 raft_consensus.cc:3037] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.976732 4712 raft_consensus.cc:482] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:45.976804 4517 raft_consensus.cc:504] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.976893 4712 raft_consensus.cc:504] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.977567 4517 leader_election.cc:288] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:45.977646 4712 leader_election.cc:288] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:45.978060 3281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e21419bf69e746baa7a96fbf5eee384d" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:45.978135 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6da0f7473ef248f2ad4f5621f3fbd25e" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:45.978266 3281 raft_consensus.cc:3037] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.978385 3149 raft_consensus.cc:2454] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:45.978600 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6da0f7473ef248f2ad4f5621f3fbd25e" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:45.978595 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e21419bf69e746baa7a96fbf5eee384d" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:45.978786 3539 raft_consensus.cc:2454] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:45.978809 3411 raft_consensus.cc:3037] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.979211 3216 leader_election.cc:302] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:45.979569 4712 raft_consensus.cc:2781] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:45.979692 4712 raft_consensus.cc:482] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:45.979781 4712 raft_consensus.cc:3037] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.981125 3281 raft_consensus.cc:2454] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:45.981464 3411 raft_consensus.cc:2454] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:45.981688 3477 leader_election.cc:302] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:45.981931 4712 raft_consensus.cc:504] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.982097 4517 raft_consensus.cc:2781] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:45.982223 4517 raft_consensus.cc:686] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:45.982564 4712 leader_election.cc:288] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:45.982481 4517 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:45.982882 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6da0f7473ef248f2ad4f5621f3fbd25e" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:45.983101 3150 raft_consensus.cc:3037] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.984124 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6da0f7473ef248f2ad4f5621f3fbd25e" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:45.984313 3539 raft_consensus.cc:3037] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:45.985698 3150 raft_consensus.cc:2454] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:45.985769 3001 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:45.986186 3216 leader_election.cc:302] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:45.986451 4712 raft_consensus.cc:2781] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:45.986655 4712 raft_consensus.cc:686] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:45.986925 4712 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:45.987792 3539 raft_consensus.cc:2454] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:45.991166 3001 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } }
I20240501 08:12:46.030292 4571 raft_consensus.cc:482] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:46.030495 4571 raft_consensus.cc:504] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.031191 4571 leader_election.cc:288] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:46.031759 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fdeb6b683c7f4097ba3054ab8d4fb203" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:46.032147 3410 raft_consensus.cc:2454] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:46.032404 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fdeb6b683c7f4097ba3054ab8d4fb203" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:46.032627 3539 raft_consensus.cc:2454] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:46.033038 3086 leader_election.cc:302] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:46.033341 4571 raft_consensus.cc:2781] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:46.033450 4571 raft_consensus.cc:482] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:46.033546 4571 raft_consensus.cc:3037] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.035673 4571 raft_consensus.cc:504] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.036247 4571 leader_election.cc:288] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:46.036666 3539 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fdeb6b683c7f4097ba3054ab8d4fb203" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:46.036746 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fdeb6b683c7f4097ba3054ab8d4fb203" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:46.036868 3539 raft_consensus.cc:3037] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.036974 3410 raft_consensus.cc:3037] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.039026 3539 raft_consensus.cc:2454] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:46.039397 3086 leader_election.cc:302] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:46.039669 4571 raft_consensus.cc:2781] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:46.039666 3410 raft_consensus.cc:2454] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:46.040318 4571 raft_consensus.cc:686] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Becoming Leader. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Running, Role: LEADER
I20240501 08:12:46.040624 4571 consensus_queue.cc:237] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.043709 3001 catalog_manager.cc:5484] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: term changed from 0 to 1, leader changed from <none> to 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
W20240501 08:12:46.389562 4758 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:46.389889 4758 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:46.397599 4758 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_7'
I20240501 08:12:46.462215 4712 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:46.478566 4517 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:46.481372 4517 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:46.485689 4571 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:46.490202 4546 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:46.558997 4756 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:46.562342 3474 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.131:44953, user_credentials={real_user=slave}} blocked reactor thread for 74236.6us
INSERT report
rows total: 2000
time total: 287.592 ms
time per row: 0.143796 ms
I20240501 08:12:46.823438 3004 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:46.831756 3130 tablet_service.cc:1467] Processing CreateTablet for tablet 0434084d13f740368d1daae53dee9fe2 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.832065 3519 tablet_service.cc:1467] Processing CreateTablet for tablet 0434084d13f740368d1daae53dee9fe2 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.832130 3259 tablet_service.cc:1467] Processing CreateTablet for tablet 0434084d13f740368d1daae53dee9fe2 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.832422 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0434084d13f740368d1daae53dee9fe2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.832489 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0434084d13f740368d1daae53dee9fe2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.832487 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0434084d13f740368d1daae53dee9fe2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.833653 3390 tablet_service.cc:1467] Processing CreateTablet for tablet ea2ecf61668b488c96b903993ab07b16 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.834045 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ea2ecf61668b488c96b903993ab07b16. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.836412 3389 tablet_service.cc:1467] Processing CreateTablet for tablet d56791555f6644a3bcee602299a85761 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.836781 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d56791555f6644a3bcee602299a85761. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.836743 3520 tablet_service.cc:1467] Processing CreateTablet for tablet d56791555f6644a3bcee602299a85761 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.837085 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d56791555f6644a3bcee602299a85761. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.841164 3128 tablet_service.cc:1467] Processing CreateTablet for tablet d56791555f6644a3bcee602299a85761 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.841574 3128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d56791555f6644a3bcee602299a85761. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.844493 3261 tablet_service.cc:1467] Processing CreateTablet for tablet ea2ecf61668b488c96b903993ab07b16 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.845158 3129 tablet_service.cc:1467] Processing CreateTablet for tablet ea2ecf61668b488c96b903993ab07b16 (DEFAULT_TABLE table=rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:46.845556 3129 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ea2ecf61668b488c96b903993ab07b16. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.848301 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ea2ecf61668b488c96b903993ab07b16. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:46.857347 4830 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:46.858142 4827 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:46.859900 4828 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:46.860548 4830 tablet_bootstrap.cc:654] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.861243 4828 tablet_bootstrap.cc:654] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.862764 4829 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:46.863116 4827 tablet_bootstrap.cc:654] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.864334 4829 tablet_bootstrap.cc:654] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.864621 4830 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:46.864790 4830 ts_tablet_manager.cc:1367] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240501 08:12:46.864979 4828 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:46.865123 4828 ts_tablet_manager.cc:1367] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:12:46.865537 4827 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:46.865720 4827 ts_tablet_manager.cc:1367] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.001s
I20240501 08:12:46.865583 4830 raft_consensus.cc:348] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.865810 4830 raft_consensus.cc:374] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.865904 4830 raft_consensus.cc:729] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.865835 4828 raft_consensus.cc:348] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.866019 4828 raft_consensus.cc:374] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.866111 4828 raft_consensus.cc:729] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.866173 4830 consensus_queue.cc:260] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.866355 4828 consensus_queue.cc:260] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.866883 4830 ts_tablet_manager.cc:1398] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:46.866948 4828 ts_tablet_manager.cc:1398] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:46.867161 4830 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:46.867193 4828 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:46.869292 4830 tablet_bootstrap.cc:654] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.869803 4827 raft_consensus.cc:348] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.870313 4827 raft_consensus.cc:374] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.870463 4827 raft_consensus.cc:729] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.871825 4828 tablet_bootstrap.cc:654] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.871048 4827 consensus_queue.cc:260] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.872839 4829 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:46.873057 4829 ts_tablet_manager.cc:1367] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.005s
I20240501 08:12:46.873948 4827 ts_tablet_manager.cc:1398] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.008s user 0.000s sys 0.002s
I20240501 08:12:46.873914 4829 raft_consensus.cc:348] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.874246 4827 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:46.874267 4829 raft_consensus.cc:374] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.874439 4829 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.874738 4829 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.876585 4827 tablet_bootstrap.cc:654] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.876590 4829 ts_tablet_manager.cc:1398] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:12:46.876878 4829 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:46.878742 4829 tablet_bootstrap.cc:654] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.880932 4830 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:46.881117 4830 ts_tablet_manager.cc:1367] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.004s
I20240501 08:12:46.881916 4830 raft_consensus.cc:348] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.882100 4830 raft_consensus.cc:374] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.882172 4830 raft_consensus.cc:729] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.882227 4828 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:46.882274 4829 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:46.882411 4829 ts_tablet_manager.cc:1367] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.001s
I20240501 08:12:46.882519 4828 ts_tablet_manager.cc:1367] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.003s
I20240501 08:12:46.882457 4830 consensus_queue.cc:260] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.882968 4827 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:46.883105 4830 ts_tablet_manager.cc:1398] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:46.883230 4827 ts_tablet_manager.cc:1367] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.002s
I20240501 08:12:46.883309 4828 raft_consensus.cc:348] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.883708 4828 raft_consensus.cc:374] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.883841 4828 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.883921 4829 raft_consensus.cc:348] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.883934 4827 raft_consensus.cc:348] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.884172 4827 raft_consensus.cc:374] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.884212 4829 raft_consensus.cc:374] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.884188 4828 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.884302 4827 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.884330 4829 raft_consensus.cc:729] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.884537 4827 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.884873 4828 ts_tablet_manager.cc:1398] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:46.885231 4827 ts_tablet_manager.cc:1398] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:46.885532 4827 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:46.886057 4829 consensus_queue.cc:260] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.886580 4829 ts_tablet_manager.cc:1398] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240501 08:12:46.887017 4827 tablet_bootstrap.cc:654] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:46.893076 4827 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:46.893219 4827 ts_tablet_manager.cc:1367] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.005s
I20240501 08:12:46.893776 4827 raft_consensus.cc:348] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.893934 4827 raft_consensus.cc:374] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:46.894018 4827 raft_consensus.cc:729] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:46.894213 4827 consensus_queue.cc:260] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.894708 4827 ts_tablet_manager.cc:1398] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.001s user 0.000s sys 0.002s
I20240501 08:12:46.950098 4787 raft_consensus.cc:482] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:46.950273 4787 raft_consensus.cc:504] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.950912 4787 leader_election.cc:288] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:46.951346 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0434084d13f740368d1daae53dee9fe2" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:46.951473 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0434084d13f740368d1daae53dee9fe2" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:46.951539 3150 raft_consensus.cc:2454] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:46.951702 3280 raft_consensus.cc:2454] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:46.951946 3476 leader_election.cc:302] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:46.952231 4787 raft_consensus.cc:2781] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:46.952358 4787 raft_consensus.cc:482] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:46.952445 4787 raft_consensus.cc:3037] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.954214 4787 raft_consensus.cc:504] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.954715 4787 leader_election.cc:288] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:46.955225 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0434084d13f740368d1daae53dee9fe2" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:46.955225 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0434084d13f740368d1daae53dee9fe2" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:46.955415 3280 raft_consensus.cc:3037] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.955415 3150 raft_consensus.cc:3037] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.958134 3280 raft_consensus.cc:2454] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:46.958134 3150 raft_consensus.cc:2454] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:46.958521 3476 leader_election.cc:302] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:46.958868 4787 raft_consensus.cc:2781] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:46.959022 4787 raft_consensus.cc:686] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:46.959344 4787 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:46.962893 3007 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:46.995568 4787 raft_consensus.cc:482] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:46.995779 4787 raft_consensus.cc:504] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:46.996349 4787 leader_election.cc:288] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:46.996799 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:46.996829 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:46.997046 3410 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:46.997046 3150 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:46.997437 3476 leader_election.cc:302] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:46.997686 4787 raft_consensus.cc:2781] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:46.997766 4787 raft_consensus.cc:482] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:46.997841 4787 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:46.999661 4787 raft_consensus.cc:504] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:47.000241 4787 leader_election.cc:288] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:47.000721 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:47.000743 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:47.000905 3150 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:47.000917 3410 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:47.002911 3150 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:47.003269 3476 leader_election.cc:302] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:47.003378 3410 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:47.003495 4787 raft_consensus.cc:2781] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:47.003667 4787 raft_consensus.cc:686] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:47.003911 4787 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:47.006737 3007 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:47.050107 4798 raft_consensus.cc:482] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:47.050278 4798 raft_consensus.cc:504] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:47.050941 4798 leader_election.cc:288] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:47.051316 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ea2ecf61668b488c96b903993ab07b16" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:47.051487 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ea2ecf61668b488c96b903993ab07b16" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:47.051559 3410 raft_consensus.cc:2454] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:47.051726 3280 raft_consensus.cc:2454] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 0.
I20240501 08:12:47.051931 3083 leader_election.cc:302] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:12:47.052211 4798 raft_consensus.cc:2781] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:47.052326 4798 raft_consensus.cc:482] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:47.052404 4798 raft_consensus.cc:3037] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:47.055007 4798 raft_consensus.cc:504] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:47.055518 4798 leader_election.cc:288] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:47.055995 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ea2ecf61668b488c96b903993ab07b16" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:47.055994 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ea2ecf61668b488c96b903993ab07b16" candidate_uuid: "466a9d70da1b4b1e8579e40447ce0c25" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:47.056210 3280 raft_consensus.cc:3037] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:47.056212 3410 raft_consensus.cc:3037] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:47.058866 3280 raft_consensus.cc:2454] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:47.058866 3410 raft_consensus.cc:2454] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466a9d70da1b4b1e8579e40447ce0c25 in term 1.
I20240501 08:12:47.059298 3086 leader_election.cc:302] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:47.059562 4798 raft_consensus.cc:2781] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:47.059691 4798 raft_consensus.cc:686] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Becoming Leader. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Running, Role: LEADER
I20240501 08:12:47.060077 4798 consensus_queue.cc:237] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } }
I20240501 08:12:47.063100 3007 catalog_manager.cc:5484] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: term changed from 0 to 1, leader changed from <none> to 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:47.387434 4787 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:47.390457 4789 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:47.409113 4787 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:47.413254 4787 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:47.441350 4798 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:47.447088 4798 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:47.451807 4843 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:47.452172 4843 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:47.459820 4843 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_8'
INSERT report
rows total: 2000
time total: 218.965 ms
time per row: 0.109482 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:12:48.688130 3005 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34812:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:12:48.696918 3390 tablet_service.cc:1467] Processing CreateTablet for tablet bdad8a9a16414c5698f9fafb7272bdb1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.696905 3259 tablet_service.cc:1467] Processing CreateTablet for tablet bdad8a9a16414c5698f9fafb7272bdb1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.697324 3390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bdad8a9a16414c5698f9fafb7272bdb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.697352 3259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bdad8a9a16414c5698f9fafb7272bdb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.698590 3520 tablet_service.cc:1467] Processing CreateTablet for tablet bdad8a9a16414c5698f9fafb7272bdb1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.698948 3520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bdad8a9a16414c5698f9fafb7272bdb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.700253 3261 tablet_service.cc:1467] Processing CreateTablet for tablet 36c9ad94ddc54bc6ab5021a0d297a774 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.700527 3261 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36c9ad94ddc54bc6ab5021a0d297a774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.704326 3519 tablet_service.cc:1467] Processing CreateTablet for tablet 36c9ad94ddc54bc6ab5021a0d297a774 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.704396 3260 tablet_service.cc:1467] Processing CreateTablet for tablet babe0d774b24482dab935eccb99f31c1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.704689 3260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet babe0d774b24482dab935eccb99f31c1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.704717 3519 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36c9ad94ddc54bc6ab5021a0d297a774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.708398 3130 tablet_service.cc:1467] Processing CreateTablet for tablet babe0d774b24482dab935eccb99f31c1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.708832 3130 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet babe0d774b24482dab935eccb99f31c1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.712888 3389 tablet_service.cc:1467] Processing CreateTablet for tablet 36c9ad94ddc54bc6ab5021a0d297a774 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.713743 3521 tablet_service.cc:1467] Processing CreateTablet for tablet babe0d774b24482dab935eccb99f31c1 (DEFAULT_TABLE table=rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:12:48.714056 3389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36c9ad94ddc54bc6ab5021a0d297a774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.714529 3521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet babe0d774b24482dab935eccb99f31c1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:48.714725 4912 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:48.716246 4912 tablet_bootstrap.cc:654] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.721516 4916 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25: Bootstrap starting.
I20240501 08:12:48.723268 4916 tablet_bootstrap.cc:654] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.724303 4915 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:48.726310 4915 tablet_bootstrap.cc:654] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.728049 4913 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:48.728528 4916 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25: No bootstrap required, opened a new log
I20240501 08:12:48.728665 4916 ts_tablet_manager.cc:1367] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240501 08:12:48.729120 4915 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:48.729329 4915 ts_tablet_manager.cc:1367] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.001s
I20240501 08:12:48.729461 4916 raft_consensus.cc:348] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.729662 4916 raft_consensus.cc:374] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.729758 4916 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466a9d70da1b4b1e8579e40447ce0c25, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.730005 4913 tablet_bootstrap.cc:654] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.729972 4916 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.730183 4915 raft_consensus.cc:348] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.730402 4915 raft_consensus.cc:374] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.730516 4916 ts_tablet_manager.cc:1398] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:48.730530 4915 raft_consensus.cc:729] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.731057 4915 consensus_queue.cc:260] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.731173 4912 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:48.731308 4912 ts_tablet_manager.cc:1367] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.003s
I20240501 08:12:48.731750 4915 ts_tablet_manager.cc:1398] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:12:48.732079 4915 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:48.732030 4912 raft_consensus.cc:348] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.732318 4912 raft_consensus.cc:374] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.732443 4912 raft_consensus.cc:729] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.732689 4912 consensus_queue.cc:260] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.733377 4912 ts_tablet_manager.cc:1398] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:12:48.733776 4912 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:48.734751 4915 tablet_bootstrap.cc:654] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.737146 4912 tablet_bootstrap.cc:654] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.738616 4913 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:48.738822 4913 ts_tablet_manager.cc:1367] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240501 08:12:48.739801 4913 raft_consensus.cc:348] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.740070 4913 raft_consensus.cc:374] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.740166 4913 raft_consensus.cc:729] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.740392 4913 consensus_queue.cc:260] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.741639 4913 ts_tablet_manager.cc:1398] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:48.741787 4912 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:48.741993 4912 ts_tablet_manager.cc:1367] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.005s
I20240501 08:12:48.742334 4915 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:48.742437 4913 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9: Bootstrap starting.
I20240501 08:12:48.742491 4915 ts_tablet_manager.cc:1367] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.003s
I20240501 08:12:48.743160 4915 raft_consensus.cc:348] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.743371 4915 raft_consensus.cc:374] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.743446 4915 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.743664 4915 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.743916 4913 tablet_bootstrap.cc:654] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.744243 4915 ts_tablet_manager.cc:1398] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:12:48.744513 4915 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: Bootstrap starting.
I20240501 08:12:48.744400 4912 raft_consensus.cc:348] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.744606 4912 raft_consensus.cc:374] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.744716 4912 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.744982 4912 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.746048 4915 tablet_bootstrap.cc:654] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.747759 4912 ts_tablet_manager.cc:1398] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240501 08:12:48.748067 4912 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Bootstrap starting.
I20240501 08:12:48.748256 4915 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: No bootstrap required, opened a new log
I20240501 08:12:48.748378 4915 ts_tablet_manager.cc:1367] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:48.748984 4915 raft_consensus.cc:348] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.749198 4915 raft_consensus.cc:374] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.749240 4913 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9: No bootstrap required, opened a new log
I20240501 08:12:48.749280 4915 raft_consensus.cc:729] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.749429 4913 ts_tablet_manager.cc:1367] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:12:48.749468 4915 consensus_queue.cc:260] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.749686 4912 tablet_bootstrap.cc:654] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Neither blocks nor log segments found. Creating new log.
I20240501 08:12:48.749946 4915 ts_tablet_manager.cc:1398] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240501 08:12:48.750677 4913 raft_consensus.cc:348] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.750841 4913 raft_consensus.cc:374] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.750936 4913 raft_consensus.cc:729] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.751188 4913 consensus_queue.cc:260] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.751673 4912 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: No bootstrap required, opened a new log
I20240501 08:12:48.751791 4913 ts_tablet_manager.cc:1398] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:48.751837 4912 ts_tablet_manager.cc:1367] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:48.752593 4912 raft_consensus.cc:348] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.752781 4912 raft_consensus.cc:374] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:12:48.752876 4912 raft_consensus.cc:729] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Initialized, Role: FOLLOWER
I20240501 08:12:48.753141 4912 consensus_queue.cc:260] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.753644 4912 ts_tablet_manager.cc:1398] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:48.763093 4925 raft_consensus.cc:482] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:48.763216 4925 raft_consensus.cc:504] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.763779 4925 leader_election.cc:288] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:48.764158 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bdad8a9a16414c5698f9fafb7272bdb1" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:48.764189 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bdad8a9a16414c5698f9fafb7272bdb1" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa" is_pre_election: true
I20240501 08:12:48.764369 3411 raft_consensus.cc:2454] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:48.764386 3541 raft_consensus.cc:2454] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 0.
I20240501 08:12:48.764787 3217 leader_election.cc:302] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:48.765022 4925 raft_consensus.cc:2781] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:48.765134 4925 raft_consensus.cc:482] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:48.765216 4925 raft_consensus.cc:3037] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.767107 4925 raft_consensus.cc:504] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.767562 4925 leader_election.cc:288] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:48.767894 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bdad8a9a16414c5698f9fafb7272bdb1" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:48.768101 3411 raft_consensus.cc:3037] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.768123 3541 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bdad8a9a16414c5698f9fafb7272bdb1" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:12:48.768311 3541 raft_consensus.cc:3037] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.770817 3541 raft_consensus.cc:2454] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:48.770807 3411 raft_consensus.cc:2454] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 1.
I20240501 08:12:48.771183 3217 leader_election.cc:302] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:48.771478 4925 raft_consensus.cc:2781] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:48.771600 4925 raft_consensus.cc:686] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:48.771842 4925 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.774487 3006 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 0 to 1, leader changed from <none> to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:48.835593 4907 raft_consensus.cc:482] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:48.835776 4907 raft_consensus.cc:504] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.836416 4907 leader_election.cc:288] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:48.836853 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36c9ad94ddc54bc6ab5021a0d297a774" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:48.836849 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36c9ad94ddc54bc6ab5021a0d297a774" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9" is_pre_election: true
I20240501 08:12:48.837069 3280 raft_consensus.cc:2454] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:48.837112 3411 raft_consensus.cc:2454] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:48.837489 3474 leader_election.cc:302] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9; no voters:
I20240501 08:12:48.837740 4907 raft_consensus.cc:2781] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:48.837839 4907 raft_consensus.cc:482] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:48.837917 4907 raft_consensus.cc:3037] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.839727 4907 raft_consensus.cc:504] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.840241 4907 leader_election.cc:288] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:48.840633 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36c9ad94ddc54bc6ab5021a0d297a774" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:48.840659 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36c9ad94ddc54bc6ab5021a0d297a774" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:48.840826 3280 raft_consensus.cc:3037] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.840868 3411 raft_consensus.cc:3037] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.842573 3280 raft_consensus.cc:2454] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:48.842645 3411 raft_consensus.cc:2454] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:48.842936 3477 leader_election.cc:302] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:48.843369 4907 raft_consensus.cc:2781] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:48.843513 4907 raft_consensus.cc:686] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:48.843773 4907 consensus_queue.cc:237] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } }
I20240501 08:12:48.846355 3006 catalog_manager.cc:5484] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:48.861615 4907 raft_consensus.cc:482] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:12:48.861786 4907 raft_consensus.cc:504] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.862296 4907 leader_election.cc:288] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:48.862701 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" is_pre_election: true
I20240501 08:12:48.862805 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25" is_pre_election: true
I20240501 08:12:48.862900 3280 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:48.863011 3150 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 0.
I20240501 08:12:48.863240 3477 leader_election.cc:302] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:48.863512 4907 raft_consensus.cc:2781] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:12:48.863597 4907 raft_consensus.cc:482] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:12:48.863687 4907 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.865463 4907 raft_consensus.cc:504] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.865962 4907 leader_election.cc:288] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:48.866252 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:12:48.866364 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:48.866452 3280 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.866551 3150 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:12:48.868191 3280 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:48.868539 3477 leader_election.cc:302] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:48.868808 4907 raft_consensus.cc:2781] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:12:48.868831 3150 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 1.
I20240501 08:12:48.868952 4907 raft_consensus.cc:686] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:12:48.869235 4907 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } }
I20240501 08:12:48.872009 3006 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } }
W20240501 08:12:49.254516 4929 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:49.254819 4929 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:12:49.258648 4925 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:49.259387 4787 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:12:49.264968 4929 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:12:49.269686 4787 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:49.297214 4939 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:12:49.323150 4937 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_9'
I20240501 08:12:49.327647 4787 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 248.146 ms
time per row: 0.124073 ms
I20240501 08:12:49.686453 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.133:43351
--local_ip_for_outbound_sockets=127.28.1.133
--tserver_master_addrs=127.28.1.190:38185
--webserver_port=33551
--webserver_interface=127.28.1.133
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240501 08:12:50.012864 4997 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:50.013127 4997 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:50.013271 4997 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:50.020782 4997 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:50.020896 4997 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:50.021045 4997 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.133
I20240501 08:12:50.030162 4997 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.133:43351
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.28.1.133
--webserver_port=33551
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.133
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:50.031435 4997 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:50.033237 4997 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:50.046761 5002 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:50.047154 5003 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:50.049687 5005 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:50.050115 4997 server_base.cc:1034] running on GCE node
I20240501 08:12:50.050679 4997 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:50.051553 4997 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:50.052747 4997 hybrid_clock.cc:648] HybridClock initialized: now 1714551170052687 us; error 55 us; skew 500 ppm
I20240501 08:12:50.053037 4997 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:50.055188 4997 webserver.cc:460] Webserver started at http://127.28.1.133:33551/ using document root <none> and password file <none>
I20240501 08:12:50.055882 4997 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:50.056038 4997 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:50.061525 4997 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.004s sys 0.000s
I20240501 08:12:50.064208 5011 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.065415 4997 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:12:50.065533 4997 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "f0af5c767c1b496cb1163870b60b066a"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:50.066259 4997 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:50.093945 4997 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:50.094741 4997 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:50.094982 4997 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:50.095832 4997 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:50.097301 4997 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:50.097371 4997 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.097460 4997 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:50.097504 4997 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.130195 4997 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.133:43351
I20240501 08:12:50.130239 5123 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.133:43351 every 8 connection(s)
I20240501 08:12:50.131997 4997 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240501 08:12:50.133658 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 4997
I20240501 08:12:50.134378 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.134:44365
--local_ip_for_outbound_sockets=127.28.1.134
--tserver_master_addrs=127.28.1.190:38185
--webserver_port=39311
--webserver_interface=127.28.1.134
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:50.150883 5124 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:50.151175 5124 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:50.151943 5124 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:50.153414 3004 ts_manager.cc:194] Re-registered known tserver with Master: f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351)
I20240501 08:12:50.154217 3004 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.133:56113
W20240501 08:12:50.486176 5128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:50.486466 5128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:50.486573 5128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:50.494251 5128 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:50.494379 5128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:50.494508 5128 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.134
I20240501 08:12:50.504477 5128 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.134:44365
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.28.1.134
--webserver_port=39311
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.134
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:50.505721 5128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:50.507725 5128 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:50.519527 5134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:50.519763 5135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:50.520073 5137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:50.520582 5128 server_base.cc:1034] running on GCE node
I20240501 08:12:50.521241 5128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:50.522256 5128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:50.523463 5128 hybrid_clock.cc:648] HybridClock initialized: now 1714551170523411 us; error 44 us; skew 500 ppm
I20240501 08:12:50.523794 5128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:50.526255 5128 webserver.cc:460] Webserver started at http://127.28.1.134:39311/ using document root <none> and password file <none>
I20240501 08:12:50.526962 5128 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:50.527107 5128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:50.532333 5128 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.001s
I20240501 08:12:50.535297 5143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.536688 5128 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:12:50.536835 5128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "8151eaf288354d60b73b3c090dd271fd"
format_stamp: "Formatted at 2024-05-01 08:12:30 on dist-test-slave-jj4n"
I20240501 08:12:50.537657 5128 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:50.593493 5128 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:50.594336 5128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:50.594637 5128 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:50.595681 5128 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:50.597224 5128 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:50.597306 5128 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.597406 5128 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:50.597452 5128 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:50.631192 5128 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.134:44365
I20240501 08:12:50.631278 5255 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.134:44365 every 8 connection(s)
I20240501 08:12:50.632887 5128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240501 08:12:50.633500 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 5128
I20240501 08:12:50.634207 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.135:42375
--local_ip_for_outbound_sockets=127.28.1.135
--tserver_master_addrs=127.28.1.190:38185
--webserver_port=33231
--webserver_interface=127.28.1.135
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:50.645633 5256 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:50.645979 5256 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:50.646941 5256 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:50.648340 3004 ts_manager.cc:194] Re-registered known tserver with Master: 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365)
I20240501 08:12:50.649170 3004 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.134:49609
W20240501 08:12:50.970762 5259 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:50.971082 5259 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:50.971236 5259 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:50.978780 5259 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:50.978899 5259 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:50.979055 5259 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.135
I20240501 08:12:50.987957 5259 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.135:42375
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.28.1.135
--webserver_port=33231
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.135
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:50.989298 5259 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:50.991012 5259 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:51.003067 5265 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:51.003005 5266 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:51.004050 5259 server_base.cc:1034] running on GCE node
W20240501 08:12:51.004071 5268 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:51.004696 5259 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:51.005606 5259 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:51.006793 5259 hybrid_clock.cc:648] HybridClock initialized: now 1714551171006743 us; error 38 us; skew 500 ppm
I20240501 08:12:51.007119 5259 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:51.009395 5259 webserver.cc:460] Webserver started at http://127.28.1.135:33231/ using document root <none> and password file <none>
I20240501 08:12:51.010089 5259 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:51.010208 5259 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:51.015393 5259 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240501 08:12:51.018399 5274 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:51.019822 5259 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:12:51.019984 5259 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "3e6a44464d7748fa8c784ef795b896fb"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:51.020823 5259 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:51.052000 5259 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:51.052820 5259 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:51.053086 5259 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:51.054049 5259 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:51.055801 5259 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:51.055876 5259 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:51.056010 5259 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:51.056070 5259 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
I20240501 08:12:51.087492 5259 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.135:42375
I20240501 08:12:51.087532 5386 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.135:42375 every 8 connection(s)
I20240501 08:12:51.088976 5259 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240501 08:12:51.094566 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 5259
I20240501 08:12:51.095271 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.136:42523
--local_ip_for_outbound_sockets=127.28.1.136
--tserver_master_addrs=127.28.1.190:38185
--webserver_port=37315
--webserver_interface=127.28.1.136
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:51.103328 5387 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:51.103636 5387 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:51.104477 5387 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:51.106406 3004 ts_manager.cc:194] Re-registered known tserver with Master: 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:12:51.107342 3004 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.135:44421
I20240501 08:12:51.156545 5124 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:51.457355 5390 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:51.457731 5390 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:51.457893 5390 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:51.467200 5390 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:51.467381 5390 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:51.467592 5390 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.136
I20240501 08:12:51.478408 5390 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.136:42523
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.28.1.136
--webserver_port=37315
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.136
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:51.479693 5390 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:51.481688 5390 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:51.493263 5397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:51.493710 5398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:51.494201 5400 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:51.494817 5390 server_base.cc:1034] running on GCE node
I20240501 08:12:51.495357 5390 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:51.496214 5390 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:51.497399 5390 hybrid_clock.cc:648] HybridClock initialized: now 1714551171497355 us; error 36 us; skew 500 ppm
I20240501 08:12:51.497699 5390 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:51.499722 5390 webserver.cc:460] Webserver started at http://127.28.1.136:37315/ using document root <none> and password file <none>
I20240501 08:12:51.500432 5390 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:51.500547 5390 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:51.505889 5390 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.002s sys 0.002s
I20240501 08:12:51.508971 5406 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:51.510383 5390 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240501 08:12:51.510542 5390 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
format_stamp: "Formatted at 2024-05-01 08:12:31 on dist-test-slave-jj4n"
I20240501 08:12:51.511365 5390 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:51.539484 5390 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:51.540365 5390 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:51.540655 5390 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:51.541678 5390 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:51.543170 5390 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:51.543241 5390 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:51.543354 5390 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:51.543399 5390 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:51.572028 5390 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.136:42523
I20240501 08:12:51.572106 5518 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.136:42523 every 8 connection(s)
I20240501 08:12:51.573539 5390 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240501 08:12:51.576707 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 5390
I20240501 08:12:51.599628 5519 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:51.599998 5519 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:51.600962 5519 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:51.602530 3004 ts_manager.cc:194] Re-registered known tserver with Master: b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523)
I20240501 08:12:51.603452 3004 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.136:41089
I20240501 08:12:51.652069 5256 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
W20240501 08:12:51.928486 5523 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:51.928905 5523 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:51.937839 5523 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:12:52.109970 5387 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
I20240501 08:12:52.230957 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:52.262737 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:52.272375 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:52.312932 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.322285 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:52.356384 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.382948 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.435601 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:12:52.438030 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.449856 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.451761 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:52.452747 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.456386 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:52.494751 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.504534 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:12:52.519026 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.606405 5519 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
I20240501 08:12:52.770144 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:52.783891 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:52.796713 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.817734 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:52.846593 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.853209 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.884313 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:52.921486 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:12:53.178007 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:53.180265 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:53.205889 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.239539 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.243940 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.246285 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:53.274736 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.275149 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
W20240501 08:12:53.351667 5655 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:53.352125 5655 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:53.363070 5655 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:12:53.409121 3538 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:53.412389 3281 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.413213 5395 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.418392 3150 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.418470 5395 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
W20240501 08:12:53.419760 3474 consensus_peers.cc:485] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: babe0d774b24482dab935eccb99f31c1. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.420212 3281 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.424536 5722 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240501 08:12:53.426716 3006 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.440701 3150 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.461201 3281 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:53.464548 3538 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.465471 3150 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.466225 5695 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.467186 5695 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.484722 5391 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.493201 3148 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.499442 3001 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.518911 3538 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
W20240501 08:12:53.530187 3215 consensus_peers.cc:485] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4cb5c6a514354708bff63dd5e51c7420. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.551744 3281 consensus_queue.cc:237] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:53.554550 3538 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.555192 3411 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.555155 5695 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.555833 5695 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:53.573854 3216 consensus_peers.cc:485] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7667242ae1c548ba982e74e190f2af27. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.591331 5747 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.598124 3538 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.597133 3411 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.606007 3001 catalog_manager.cc:5484] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.661777 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:53.676918 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:53.683287 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.683490 3538 consensus_queue.cc:237] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:53.695533 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:53.741457 3411 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.742205 5725 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:53.749620 3475 consensus_peers.cc:485] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6e2593a1a02d4f61972ba63e7dbc628c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.750195 3149 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.752672 5725 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.756542 5725 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:53.760563 3149 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:53.766484 3007 catalog_manager.cc:5484] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:53.774319 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.794687 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:12:53.797425 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.815605 3411 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:53.839809 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:53.879671 3538 consensus_queue.cc:237] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:53.882508 3149 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.882545 3281 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:53.882896 3474 consensus_peers.cc:485] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0612f1f8e4ca4a29971151bf52571a33. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.883174 5726 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.883432 5725 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.891295 5725 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.892136 3281 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.893035 3149 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:53.897460 3007 catalog_manager.cc:5484] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:53.915542 3281 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:53.918113 3411 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.918208 3538 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:53.918215 3215 consensus_peers.cc:485] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7fde726fbe8d48f685a4580ff6ae4033. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.918763 5695 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.919060 5391 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.922009 5391 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.922674 3538 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.925798 3411 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:53.927246 3007 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:53.946415 3149 consensus_queue.cc:237] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:53.951229 3281 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.951414 3411 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.951738 5532 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.952445 5811 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.954792 5532 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.955209 3281 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:53.962261 3003 catalog_manager.cc:5484] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:53.970299 3411 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
W20240501 08:12:53.980934 3085 consensus_peers.cc:485] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ea2ecf61668b488c96b903993ab07b16. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:53.992545 3281 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:53.996033 3538 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.996033 3149 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:53.996690 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:53.997555 5695 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.004699 5821 ts_tablet_manager.cc:902] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
W20240501 08:12:54.005542 3215 consensus_peers.cc:485] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d749fd4842294f8fba36f287b7072c9d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.008280 5819 ts_tablet_manager.cc:902] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.013952 5391 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.017129 3148 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.022154 3001 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.025468 3541 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.066999 5828 ts_tablet_manager.cc:902] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:54.068372 5821 tablet_copy_client.cc:308] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.084101 5819 tablet_copy_client.cc:308] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.086870 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 4cb5c6a514354708bff63dd5e51c7420 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:45681)
I20240501 08:12:54.087167 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 4cb5c6a514354708bff63dd5e51c7420 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:45681: session id = 3e6a44464d7748fa8c784ef795b896fb-4cb5c6a514354708bff63dd5e51c7420
I20240501 08:12:54.093216 3301 tablet_copy_source_session.cc:205] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.093662 5828 tablet_copy_client.cc:308] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:54.099354 5821 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cb5c6a514354708bff63dd5e51c7420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.118480 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet babe0d774b24482dab935eccb99f31c1 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:56329)
I20240501 08:12:54.118798 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet babe0d774b24482dab935eccb99f31c1 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:56329: session id = f0af5c767c1b496cb1163870b60b066a-babe0d774b24482dab935eccb99f31c1
I20240501 08:12:54.120956 5821 tablet_copy_client.cc:783] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.121341 5821 tablet_copy_client.cc:647] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.122330 3561 tablet_copy_source_session.cc:205] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.123719 5832 tablet_copy_client.cc:1002] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.124508 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 7667242ae1c548ba982e74e190f2af27 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:12:54.124699 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 7667242ae1c548ba982e74e190f2af27 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-7667242ae1c548ba982e74e190f2af27
I20240501 08:12:54.126973 3301 tablet_copy_source_session.cc:205] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.127751 5828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet babe0d774b24482dab935eccb99f31c1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.131541 5819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7667242ae1c548ba982e74e190f2af27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.137300 3541 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:54.139881 3411 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.139881 3281 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.140530 5722 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.141062 5722 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:54.146771 3475 consensus_peers.cc:485] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e21419bf69e746baa7a96fbf5eee384d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.148602 5850 ts_tablet_manager.cc:902] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:54.148689 5726 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.149447 5828 tablet_copy_client.cc:783] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.149801 5828 tablet_copy_client.cc:647] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.152793 3281 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.157014 5850 tablet_copy_client.cc:308] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:54.159822 3001 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.162104 5843 tablet_copy_client.cc:1002] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.178073 3411 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.180200 5819 tablet_copy_client.cc:783] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.180567 5819 tablet_copy_client.cc:647] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.182648 5835 tablet_copy_client.cc:1002] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.186528 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 6e2593a1a02d4f61972ba63e7dbc628c from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:60309)
I20240501 08:12:54.186725 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 6e2593a1a02d4f61972ba63e7dbc628c from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:60309: session id = b1a5f0e8d89547a1ac983c6a5256eca8-6e2593a1a02d4f61972ba63e7dbc628c
I20240501 08:12:54.189692 3561 tablet_copy_source_session.cc:205] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.195050 5850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6e2593a1a02d4f61972ba63e7dbc628c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.209192 5850 tablet_copy_client.cc:783] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.209599 5850 tablet_copy_client.cc:647] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.211562 5854 tablet_copy_client.cc:1002] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.230521 3281 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:54.233155 3148 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.233831 5695 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.234038 3541 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.234645 5695 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.237629 5695 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
W20240501 08:12:54.239614 3214 consensus_peers.cc:485] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d110b6fb2855454892ab5359ec3210b0. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.241473 3541 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.241994 3148 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.245784 3001 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.263202 3148 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:54.265758 3411 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.266963 5532 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.267160 3281 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.268138 5532 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:54.273896 3084 consensus_peers.cc:485] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a3fce8868ca74a028492122fc2762001. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.279453 5532 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.279932 3411 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.282903 3280 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.285396 3000 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.308624 3541 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:54.311813 3148 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.312927 5725 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.313196 3280 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.315577 5722 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.316149 5726 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.317688 3148 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
W20240501 08:12:54.319924 3476 consensus_peers.cc:485] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a68f4cc374ad4656b1de788033ece98c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.323890 3000 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.327997 3280 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.350111 3280 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
W20240501 08:12:54.352715 3215 consensus_peers.cc:485] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f0c4a54ade9c4e96bb575bfbfa5d0206. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.352764 3411 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.353293 5391 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.353693 3148 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.354290 5391 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.356679 5747 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.357326 3410 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.358289 3149 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.363760 3000 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.380308 3280 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:54.382920 3541 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:54.383277 3214 consensus_peers.cc:485] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 460948db7dfd4d5589d478ddd5c6c10c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.383430 3410 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.384039 5391 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.384354 5695 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.387477 5747 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.388111 3541 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.391050 3410 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.392630 3000 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.417495 3410 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:54.420552 3148 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.421272 3280 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.421844 5664 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.422430 5664 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.439348 5664 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
W20240501 08:12:54.447567 3345 consensus_peers.cc:485] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c927478b05c14f47b736faf52b4db9aa. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.448522 5904 ts_tablet_manager.cc:902] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:54.448328 3148 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.455204 3003 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.479758 3280 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.504770 3148 consensus_queue.cc:237] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
W20240501 08:12:54.513541 3085 consensus_peers.cc:485] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fdeb6b683c7f4097ba3054ab8d4fb203. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.514700 3541 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.515372 5532 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.523113 3410 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.523978 5532 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.538745 5920 ts_tablet_manager.cc:902] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.542766 5532 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.547513 3411 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.553614 5920 tablet_copy_client.cc:308] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.554440 5904 tablet_copy_client.cc:308] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:54.554531 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 7fde726fbe8d48f685a4580ff6ae4033 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:45681)
I20240501 08:12:54.554672 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 7fde726fbe8d48f685a4580ff6ae4033 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:45681: session id = 3e6a44464d7748fa8c784ef795b896fb-7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:12:54.557790 3301 tablet_copy_source_session.cc:205] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.558692 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 0612f1f8e4ca4a29971151bf52571a33 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:56329)
I20240501 08:12:54.558866 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 0612f1f8e4ca4a29971151bf52571a33 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:56329: session id = f0af5c767c1b496cb1163870b60b066a-0612f1f8e4ca4a29971151bf52571a33
I20240501 08:12:54.561807 3561 tablet_copy_source_session.cc:205] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.563189 3007 catalog_manager.cc:5484] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.565975 5904 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0612f1f8e4ca4a29971151bf52571a33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.572147 5904 tablet_copy_client.cc:783] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.572409 5904 tablet_copy_client.cc:647] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.579267 5923 tablet_copy_client.cc:1002] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.587692 5920 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fde726fbe8d48f685a4580ff6ae4033. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.591070 5930 ts_tablet_manager.cc:902] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:54.600926 3538 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:54.633813 5920 tablet_copy_client.cc:783] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.634105 5920 tablet_copy_client.cc:647] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.634670 5936 ts_tablet_manager.cc:902] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.635661 5934 ts_tablet_manager.cc:902] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:54.635826 5927 tablet_copy_client.cc:1002] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.637693 5936 tablet_copy_client.cc:308] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.638093 5934 tablet_copy_client.cc:308] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:54.645882 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet d749fd4842294f8fba36f287b7072c9d from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:42911)
I20240501 08:12:54.645931 5930 tablet_copy_client.cc:308] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:12:54.646082 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet d749fd4842294f8fba36f287b7072c9d from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:42911: session id = b1a5f0e8d89547a1ac983c6a5256eca8-d749fd4842294f8fba36f287b7072c9d
I20240501 08:12:54.649034 3301 tablet_copy_source_session.cc:205] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.650717 5936 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d749fd4842294f8fba36f287b7072c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.657373 5936 tablet_copy_client.cc:783] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.657681 5936 tablet_copy_client.cc:647] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.660391 5947 tablet_copy_client.cc:1002] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.662393 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet e21419bf69e746baa7a96fbf5eee384d from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:44357)
I20240501 08:12:54.662567 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet e21419bf69e746baa7a96fbf5eee384d from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:44357: session id = 3e6a44464d7748fa8c784ef795b896fb-e21419bf69e746baa7a96fbf5eee384d
I20240501 08:12:54.665302 3561 tablet_copy_source_session.cc:205] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.667088 5934 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e21419bf69e746baa7a96fbf5eee384d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.668020 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet ea2ecf61668b488c96b903993ab07b16 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:35817)
I20240501 08:12:54.668289 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet ea2ecf61668b488c96b903993ab07b16 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:35817: session id = 8151eaf288354d60b73b3c090dd271fd-ea2ecf61668b488c96b903993ab07b16
I20240501 08:12:54.673360 5934 tablet_copy_client.cc:783] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.673602 5934 tablet_copy_client.cc:647] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.674348 3170 tablet_copy_source_session.cc:205] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.675791 5948 tablet_copy_client.cc:1002] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.677414 5930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ea2ecf61668b488c96b903993ab07b16. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.680517 3410 consensus_queue.cc:237] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
W20240501 08:12:54.681289 5846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:54.681702 5846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:12:54.682731 5930 tablet_copy_client.cc:783] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.682961 5930 tablet_copy_client.cc:647] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.683728 3541 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.684911 3148 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.685022 5664 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.685454 5906 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.686228 5945 tablet_copy_client.cc:1002] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.687834 5906 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.691521 3541 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
W20240501 08:12:54.692777 5846 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
W20240501 08:12:54.693790 3345 consensus_peers.cc:485] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d3db152c64e74ee3a788769dba03b7a6. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.697286 3000 catalog_manager.cc:5484] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.703492 3148 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:54.729367 3410 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:54.731598 3148 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.731734 3280 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:54.732214 3345 consensus_peers.cc:485] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 885de7b67f474f7bb54badfae193906d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.732685 5906 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.732981 5664 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.736322 5906 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.737568 3148 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.741568 3280 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:54.742269 3000 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.745811 5974 ts_tablet_manager.cc:902] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:54.773416 3541 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:54.776095 3410 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:54.776327 3148 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:54.776513 3474 consensus_peers.cc:485] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d56791555f6644a3bcee602299a85761. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:54.776835 5726 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.777293 5879 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:54.780539 5726 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.783658 3148 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.787756 3410 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:54.787730 3000 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:54.807207 5974 tablet_copy_client.cc:308] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:12:54.813441 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet a3fce8868ca74a028492122fc2762001 from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:35605)
I20240501 08:12:54.813632 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet a3fce8868ca74a028492122fc2762001 from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:35605: session id = b1a5f0e8d89547a1ac983c6a5256eca8-a3fce8868ca74a028492122fc2762001
I20240501 08:12:54.817174 3170 tablet_copy_source_session.cc:205] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.820420 5974 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a3fce8868ca74a028492122fc2762001. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.826668 5974 tablet_copy_client.cc:783] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.826921 5974 tablet_copy_client.cc:647] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.829058 5987 tablet_copy_client.cc:1002] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.829754 6002 ts_tablet_manager.cc:902] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.850615 6002 tablet_copy_client.cc:308] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.861021 6010 ts_tablet_manager.cc:902] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.865203 6010 tablet_copy_client.cc:308] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.870294 3300 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet d110b6fb2855454892ab5359ec3210b0 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:47763)
I20240501 08:12:54.870507 3300 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet d110b6fb2855454892ab5359ec3210b0 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:47763: session id = f0af5c767c1b496cb1163870b60b066a-d110b6fb2855454892ab5359ec3210b0
I20240501 08:12:54.872083 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 460948db7dfd4d5589d478ddd5c6c10c from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:47763)
I20240501 08:12:54.872241 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 460948db7dfd4d5589d478ddd5c6c10c from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:47763: session id = f0af5c767c1b496cb1163870b60b066a-460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:12:54.872861 3300 tablet_copy_source_session.cc:205] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.874188 6010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d110b6fb2855454892ab5359ec3210b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.874807 3301 tablet_copy_source_session.cc:205] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.880002 6002 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 460948db7dfd4d5589d478ddd5c6c10c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.888350 6002 tablet_copy_client.cc:783] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.888773 6002 tablet_copy_client.cc:647] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.890612 6015 tablet_copy_client.cc:1002] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.891436 6010 tablet_copy_client.cc:783] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.891744 6010 tablet_copy_client.cc:647] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.893496 6022 tablet_copy_client.cc:1002] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.896378 6018 ts_tablet_manager.cc:902] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:54.897755 6017 ts_tablet_manager.cc:902] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:54.902145 6018 tablet_copy_client.cc:308] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:54.903836 6017 tablet_copy_client.cc:308] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:54.907150 6035 ts_tablet_manager.cc:902] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:54.911494 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet a68f4cc374ad4656b1de788033ece98c from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:48067)
I20240501 08:12:54.911671 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet a68f4cc374ad4656b1de788033ece98c from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:48067: session id = 8151eaf288354d60b73b3c090dd271fd-a68f4cc374ad4656b1de788033ece98c
I20240501 08:12:54.914721 3561 tablet_copy_source_session.cc:205] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.916333 6018 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a68f4cc374ad4656b1de788033ece98c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.922684 6018 tablet_copy_client.cc:783] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.923005 6018 tablet_copy_client.cc:647] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.923250 3431 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet c927478b05c14f47b736faf52b4db9aa from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:60049)
I20240501 08:12:54.923503 3431 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet c927478b05c14f47b736faf52b4db9aa from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:60049: session id = b1a5f0e8d89547a1ac983c6a5256eca8-c927478b05c14f47b736faf52b4db9aa
I20240501 08:12:54.926906 3431 tablet_copy_source_session.cc:205] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.928963 6017 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c927478b05c14f47b736faf52b4db9aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:54.934703 6017 tablet_copy_client.cc:783] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:54.934980 6017 tablet_copy_client.cc:647] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:54.937265 6034 tablet_copy_client.cc:1002] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.943758 6052 ts_tablet_manager.cc:902] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:54.945868 6033 tablet_copy_client.cc:1002] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:54.947898 6035 tablet_copy_client.cc:308] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:54.949160 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:45681)
I20240501 08:12:54.949348 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet f0c4a54ade9c4e96bb575bfbfa5d0206 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:45681: session id = 3e6a44464d7748fa8c784ef795b896fb-f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:12:54.952270 3301 tablet_copy_source_session.cc:205] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:54.963541 6035 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f0c4a54ade9c4e96bb575bfbfa5d0206. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:55.065807 6035 tablet_copy_client.cc:783] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:55.066226 6035 tablet_copy_client.cc:647] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:55.068430 6054 tablet_copy_client.cc:1002] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:55.076529 6052 tablet_copy_client.cc:308] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:12:55.077332 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet fdeb6b683c7f4097ba3054ab8d4fb203 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:35817)
I20240501 08:12:55.077498 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet fdeb6b683c7f4097ba3054ab8d4fb203 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:35817: session id = 8151eaf288354d60b73b3c090dd271fd-fdeb6b683c7f4097ba3054ab8d4fb203
I20240501 08:12:55.080219 3170 tablet_copy_source_session.cc:205] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:55.082473 6052 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fdeb6b683c7f4097ba3054ab8d4fb203. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:55.090678 6052 tablet_copy_client.cc:783] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:55.090984 6052 tablet_copy_client.cc:647] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:55.092661 6063 tablet_copy_client.cc:1002] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:55.121604 3520 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:12:55.125308 3390 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:12:55.148986 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:55.194154 6075 ts_tablet_manager.cc:902] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:55.208535 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:55.220067 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:12:55.247834 3259 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:12:55.283715 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:55.300585 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:12:55.339082 6090 ts_tablet_manager.cc:902] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:55.341293 6087 ts_tablet_manager.cc:902] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:55.396060 6075 tablet_copy_client.cc:308] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:55.401314 6087 tablet_copy_client.cc:308] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:55.402280 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet d56791555f6644a3bcee602299a85761 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:56329)
I20240501 08:12:55.402501 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet d56791555f6644a3bcee602299a85761 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:56329: session id = f0af5c767c1b496cb1163870b60b066a-d56791555f6644a3bcee602299a85761
I20240501 08:12:55.405616 3561 tablet_copy_source_session.cc:205] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:55.407433 6087 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d56791555f6644a3bcee602299a85761. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:55.413573 6090 tablet_copy_client.cc:308] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:55.415359 3431 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 885de7b67f474f7bb54badfae193906d from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:60049)
I20240501 08:12:55.415549 3431 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 885de7b67f474f7bb54badfae193906d from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:60049: session id = b1a5f0e8d89547a1ac983c6a5256eca8-885de7b67f474f7bb54badfae193906d
I20240501 08:12:55.416674 6087 tablet_copy_client.cc:783] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:55.417122 6087 tablet_copy_client.cc:647] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:55.418329 3431 tablet_copy_source_session.cc:205] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:55.418560 6102 tablet_copy_client.cc:1002] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:55.421759 6090 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 885de7b67f474f7bb54badfae193906d. 1 dirs total, 0 dirs full, 0 dirs failed
Runtime error: ksck discovered errors
I20240501 08:12:55.433368 6090 tablet_copy_client.cc:783] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:55.433713 6090 tablet_copy_client.cc:647] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:55.435499 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet d3db152c64e74ee3a788769dba03b7a6 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:40907)
I20240501 08:12:55.435708 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet d3db152c64e74ee3a788769dba03b7a6 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:40907: session id = 3e6a44464d7748fa8c784ef795b896fb-d3db152c64e74ee3a788769dba03b7a6
I20240501 08:12:55.436486 6109 tablet_copy_client.cc:1002] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:55.438050 3430 tablet_copy_source_session.cc:205] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:55.439321 6075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d3db152c64e74ee3a788769dba03b7a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:55.447547 6075 tablet_copy_client.cc:783] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:55.447856 6075 tablet_copy_client.cc:647] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:55.449590 6097 tablet_copy_client.cc:1002] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:55.552359 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.137:0
--local_ip_for_outbound_sockets=127.28.1.137
--webserver_interface=127.28.1.137
--webserver_port=0
--tserver_master_addrs=127.28.1.190:38185
--builtin_ntp_servers=127.28.1.148:37195
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240501 08:12:55.624303 5821 tablet_copy_client.cc:523] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:55.628715 5821 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:55.642292 5821 log.cc:824] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:55.662650 5828 tablet_copy_client.cc:523] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:55.667130 5828 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:55.681300 5828 log.cc:824] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:55.683176 5819 tablet_copy_client.cc:523] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:55.687525 5819 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:12:55.688022 5821 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:55.688594 5821 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:55.689074 5821 ts_tablet_manager.cc:1367] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.061s user 0.054s sys 0.001s
I20240501 08:12:55.695734 5821 raft_consensus.cc:348] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:55.696276 5821 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:55.696858 5821 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:55.705090 5819 log.cc:824] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:55.706969 5821 ts_tablet_manager.cc:1398] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.018s user 0.012s sys 0.003s
I20240501 08:12:55.708051 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-4cb5c6a514354708bff63dd5e51c7420 received from {username='slave'} at 127.28.1.135:45681
I20240501 08:12:55.708185 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-4cb5c6a514354708bff63dd5e51c7420 on tablet 4cb5c6a514354708bff63dd5e51c7420 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:55.712220 5850 tablet_copy_client.cc:523] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:55.716866 5850 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:55.729691 5850 log.cc:824] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Log is configured to *not* fsync() on all Append() calls
I20240501 08:12:55.755955 5828 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:55.756669 5828 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:55.757201 5828 ts_tablet_manager.cc:1367] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.090s user 0.054s sys 0.015s
I20240501 08:12:55.767899 5819 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:55.767558 5828 raft_consensus.cc:348] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:55.768345 5828 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:55.768558 5819 tablet_bootstrap.cc:492] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:12:55.769033 5819 ts_tablet_manager.cc:1367] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.082s user 0.055s sys 0.008s
I20240501 08:12:55.769279 5828 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:55.779496 5819 raft_consensus.cc:348] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:55.780064 5819 raft_consensus.cc:729] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:12:55.780630 5819 consensus_queue.cc:260] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:55.785234 5828 ts_tablet_manager.cc:1398] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.028s user 0.014s sys 0.000s
I20240501 08:12:55.786716 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-babe0d774b24482dab935eccb99f31c1 received from {username='slave'} at 127.28.1.133:56329
I20240501 08:12:55.786880 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-babe0d774b24482dab935eccb99f31c1 on tablet babe0d774b24482dab935eccb99f31c1 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:55.790681 5819 ts_tablet_manager.cc:1398] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.022s user 0.008s sys 0.008s
I20240501 08:12:55.793076 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-7667242ae1c548ba982e74e190f2af27 received from {username='slave'} at 127.28.1.134:50227
I20240501 08:12:55.793210 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-7667242ae1c548ba982e74e190f2af27 on tablet 7667242ae1c548ba982e74e190f2af27 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:12:55.793525 5850 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:55.794613 5850 tablet_bootstrap.cc:492] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:55.795056 5850 ts_tablet_manager.cc:1367] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.078s user 0.068s sys 0.004s
I20240501 08:12:55.801003 5850 raft_consensus.cc:348] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:55.801421 5850 raft_consensus.cc:729] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:55.801858 5850 consensus_queue.cc:260] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:55.803261 5850 ts_tablet_manager.cc:1398] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.008s user 0.008s sys 0.000s
I20240501 08:12:55.805577 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-6e2593a1a02d4f61972ba63e7dbc628c received from {username='slave'} at 127.28.1.136:60309
I20240501 08:12:55.805706 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-6e2593a1a02d4f61972ba63e7dbc628c on tablet 6e2593a1a02d4f61972ba63e7dbc628c with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:55.904407 5342 raft_consensus.cc:1203] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:55.966410 5474 raft_consensus.cc:1203] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
W20240501 08:12:55.979890 6152 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:12:55.980314 6152 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:12:55.980477 6152 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:12:55.990563 6152 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:12:55.990674 6152 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:12:55.990839 6152 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.137
I20240501 08:12:56.004961 6152 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:37195
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.137:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.28.1.137
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.28.1.190:38185
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.137
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:12:56.006345 6152 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:12:56.008229 6152 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:12:56.050746 6263 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:12:56.054010 6268 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:56.054543 6152 server_base.cc:1034] running on GCE node
W20240501 08:12:56.055471 6265 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:12:56.056301 6152 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:12:56.057466 6152 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:12:56.058701 6152 hybrid_clock.cc:648] HybridClock initialized: now 1714551176058620 us; error 68 us; skew 500 ppm
I20240501 08:12:56.059129 6152 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:12:56.064560 6152 webserver.cc:460] Webserver started at http://127.28.1.137:39271/ using document root <none> and password file <none>
I20240501 08:12:56.065377 6152 fs_manager.cc:359] Metadata directory not provided
I20240501 08:12:56.065495 6152 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:12:56.065827 6152 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:12:56.068729 6152 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "53ae6b4d587b4c48a6dcc10a7ba67579"
format_stamp: "Formatted at 2024-05-01 08:12:56 on dist-test-slave-jj4n"
I20240501 08:12:56.069594 6152 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "53ae6b4d587b4c48a6dcc10a7ba67579"
format_stamp: "Formatted at 2024-05-01 08:12:56 on dist-test-slave-jj4n"
I20240501 08:12:56.075153 6152 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:12:56.079408 6279 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:56.079993 5904 tablet_copy_client.cc:523] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.081215 6152 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
I20240501 08:12:56.081363 6152 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "53ae6b4d587b4c48a6dcc10a7ba67579"
format_stamp: "Formatted at 2024-05-01 08:12:56 on dist-test-slave-jj4n"
I20240501 08:12:56.081545 6152 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:12:56.082765 5904 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:56.118849 6152 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:12:56.119922 6152 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:12:56.120251 6152 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:12:56.121398 6152 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:12:56.123473 6152 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:12:56.123586 6152 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:56.123699 6152 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:12:56.123773 6152 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:12:56.134155 5904 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.134577 5904 tablet_bootstrap.cc:492] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:56.134833 5904 ts_tablet_manager.cc:1367] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.052s user 0.039s sys 0.006s
I20240501 08:12:56.135784 5904 raft_consensus.cc:348] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.136075 5904 raft_consensus.cc:729] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:56.136292 5904 consensus_queue.cc:260] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.136863 5904 ts_tablet_manager.cc:1398] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:12:56.140314 5920 tablet_copy_client.cc:523] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.143388 5920 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:56.143834 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-0612f1f8e4ca4a29971151bf52571a33 received from {username='slave'} at 127.28.1.133:56329
I20240501 08:12:56.144021 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-0612f1f8e4ca4a29971151bf52571a33 on tablet 0612f1f8e4ca4a29971151bf52571a33 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:56.145017 5211 raft_consensus.cc:1203] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.146965 5079 raft_consensus.cc:1203] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.160816 5936 tablet_copy_client.cc:523] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.164950 5936 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:56.176175 5934 tablet_copy_client.cc:523] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.186812 5930 tablet_copy_client.cc:523] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.190569 5930 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:12:56.193164 5934 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:56.205514 5920 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.206071 5920 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:56.206368 5920 ts_tablet_manager.cc:1367] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.063s user 0.042s sys 0.009s
I20240501 08:12:56.207345 5920 raft_consensus.cc:348] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.207614 5920 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:56.207821 5920 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.208369 5920 ts_tablet_manager.cc:1398] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:12:56.209338 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-7fde726fbe8d48f685a4580ff6ae4033 received from {username='slave'} at 127.28.1.135:45681
I20240501 08:12:56.209470 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-7fde726fbe8d48f685a4580ff6ae4033 on tablet 7fde726fbe8d48f685a4580ff6ae4033 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:56.261992 5934 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.262571 5934 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:56.263029 5934 ts_tablet_manager.cc:1367] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.070s user 0.050s sys 0.000s
I20240501 08:12:56.263949 5930 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.263906 5934 raft_consensus.cc:348] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.264338 5934 raft_consensus.cc:729] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:56.264470 5930 tablet_bootstrap.cc:492] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:12:56.264638 5934 consensus_queue.cc:260] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.264811 5930 ts_tablet_manager.cc:1367] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.074s user 0.046s sys 0.002s
I20240501 08:12:56.265280 5934 ts_tablet_manager.cc:1398] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:56.266065 5930 raft_consensus.cc:348] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.266364 5930 raft_consensus.cc:729] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:12:56.266582 5930 consensus_queue.cc:260] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.267156 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-e21419bf69e746baa7a96fbf5eee384d received from {username='slave'} at 127.28.1.135:44357
I20240501 08:12:56.267284 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-e21419bf69e746baa7a96fbf5eee384d on tablet e21419bf69e746baa7a96fbf5eee384d with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:56.269767 5930 ts_tablet_manager.cc:1398] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:12:56.272842 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-ea2ecf61668b488c96b903993ab07b16 received from {username='slave'} at 127.28.1.134:35817
I20240501 08:12:56.273061 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-ea2ecf61668b488c96b903993ab07b16 on tablet ea2ecf61668b488c96b903993ab07b16 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:12:56.274958 5936 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.275310 5936 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:56.275547 5936 ts_tablet_manager.cc:1367] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.111s user 0.050s sys 0.034s
I20240501 08:12:56.276356 5936 raft_consensus.cc:348] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.276656 5936 raft_consensus.cc:729] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:56.276845 5936 consensus_queue.cc:260] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.277505 5936 ts_tablet_manager.cc:1398] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:12:56.278240 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-d749fd4842294f8fba36f287b7072c9d received from {username='slave'} at 127.28.1.136:42911
I20240501 08:12:56.278381 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-d749fd4842294f8fba36f287b7072c9d on tablet d749fd4842294f8fba36f287b7072c9d with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:56.320188 6152 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.137:42431
I20240501 08:12:56.320250 6411 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.137:42431 every 8 connection(s)
I20240501 08:12:56.321856 6152 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240501 08:12:56.322660 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 6152
I20240501 08:12:56.322865 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240501 08:12:56.329438 5974 tablet_copy_client.cc:523] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.332675 5974 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:56.340675 6412 heartbeater.cc:347] Connected to a master server at 127.28.1.190:38185
I20240501 08:12:56.340927 6412 heartbeater.cc:464] Registering TS with master...
I20240501 08:12:56.341936 6412 heartbeater.cc:511] Master 127.28.1.190:38185 requested a full tablet report, sending...
I20240501 08:12:56.343794 3000 ts_manager.cc:194] Registered new tserver with Master: 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431)
I20240501 08:12:56.344803 3000 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.137:35247
I20240501 08:12:56.354214 5342 raft_consensus.cc:1203] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.375192 5974 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.375562 5974 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:56.375797 5974 ts_tablet_manager.cc:1367] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.043s user 0.032s sys 0.005s
I20240501 08:12:56.376525 5974 raft_consensus.cc:348] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.376793 5974 raft_consensus.cc:729] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:56.377046 5974 consensus_queue.cc:260] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.377540 5974 ts_tablet_manager.cc:1398] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240501 08:12:56.378298 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-a3fce8868ca74a028492122fc2762001 received from {username='slave'} at 127.28.1.136:35605
I20240501 08:12:56.378450 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-a3fce8868ca74a028492122fc2762001 on tablet a3fce8868ca74a028492122fc2762001 with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:56.391152 6002 tablet_copy_client.cc:523] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.393853 6010 tablet_copy_client.cc:523] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.394368 6002 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:56.397478 6010 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:56.438225 6451 raft_consensus.cc:1050] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:12:56.438467 5391 raft_consensus.cc:1050] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:12:56.439240 6451 consensus_queue.cc:237] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:56.439349 5391 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:56.440047 6017 tablet_copy_client.cc:523] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.443536 6017 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:56.444666 6010 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.445154 5079 raft_consensus.cc:1203] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.445226 6010 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:56.445628 6010 ts_tablet_manager.cc:1367] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.048s user 0.040s sys 0.007s
I20240501 08:12:56.446236 6018 tablet_copy_client.cc:523] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.447468 5211 raft_consensus.cc:1203] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.450054 6018 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:12:56.456784 6002 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.457245 6002 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:56.462703 6010 raft_consensus.cc:348] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.463111 6010 raft_consensus.cc:729] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:56.463378 6010 consensus_queue.cc:260] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.463546 6002 ts_tablet_manager.cc:1367] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.069s user 0.035s sys 0.008s
I20240501 08:12:56.465934 6002 raft_consensus.cc:348] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.466190 6002 raft_consensus.cc:729] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:56.466418 6002 consensus_queue.cc:260] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.466902 6002 ts_tablet_manager.cc:1398] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:56.467528 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-460948db7dfd4d5589d478ddd5c6c10c received from {username='slave'} at 127.28.1.133:47763
I20240501 08:12:56.467650 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-460948db7dfd4d5589d478ddd5c6c10c on tablet 460948db7dfd4d5589d478ddd5c6c10c with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:56.465615 6010 ts_tablet_manager.cc:1398] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.020s user 0.002s sys 0.000s
I20240501 08:12:56.481180 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-d110b6fb2855454892ab5359ec3210b0 received from {username='slave'} at 127.28.1.133:47763
I20240501 08:12:56.481339 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-d110b6fb2855454892ab5359ec3210b0 on tablet d110b6fb2855454892ab5359ec3210b0 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:56.495725 5342 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.496502 5747 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.506868 3538 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.507472 5747 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.519057 3149 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.519987 5747 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.540257 6018 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.540793 6018 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:12:56.541157 6018 ts_tablet_manager.cc:1367] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.091s user 0.044s sys 0.009s
I20240501 08:12:56.541226 6017 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.541682 6017 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:56.542002 6017 ts_tablet_manager.cc:1367] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.099s user 0.045s sys 0.009s
I20240501 08:12:56.542289 6018 raft_consensus.cc:348] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.542627 6018 raft_consensus.cc:729] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:12:56.542897 6018 consensus_queue.cc:260] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.543699 6018 ts_tablet_manager.cc:1398] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:12:56.544860 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-a68f4cc374ad4656b1de788033ece98c received from {username='slave'} at 127.28.1.134:48067
I20240501 08:12:56.544996 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-a68f4cc374ad4656b1de788033ece98c on tablet a68f4cc374ad4656b1de788033ece98c with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:12:56.544939 6017 raft_consensus.cc:348] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.545172 6017 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:56.545910 6017 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:56.546525 6017 ts_tablet_manager.cc:1398] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:12:56.547302 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-c927478b05c14f47b736faf52b4db9aa received from {username='slave'} at 127.28.1.136:60049
I20240501 08:12:56.547487 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-c927478b05c14f47b736faf52b4db9aa on tablet c927478b05c14f47b736faf52b4db9aa with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:56.552878 5474 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.553494 5726 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.556398 3149 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.557242 6453 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.559319 5747 raft_consensus.cc:1050] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:12:56.560501 5747 consensus_queue.cc:237] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:56.562906 5079 raft_consensus.cc:1203] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.564389 6472 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.568787 6035 tablet_copy_client.cc:523] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.571348 3538 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.573333 3410 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.573995 5726 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.577030 6452 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.577983 6479 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.581225 6035 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:56.581944 3411 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.582041 3150 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.582278 5474 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.587600 3000 catalog_manager.cc:5484] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.592459 5747 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.593047 6052 tablet_copy_client.cc:523] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.593442 5474 raft_consensus.cc:1203] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.597471 6052 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:12:56.599081 5341 raft_consensus.cc:1203] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.598567 5342 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.608330 5211 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.615279 3410 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.614570 5391 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240501 08:12:56.620574 3007 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:56.625514 3539 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.639300 3150 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.660411 6479 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.661796 3410 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.672415 6459 raft_consensus.cc:1050] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:12:56.673235 6459 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:56.671900 5211 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.676507 3538 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.684762 3005 catalog_manager.cc:5484] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.692166 3539 consensus_queue.cc:237] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:56.694732 3410 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.694803 5474 raft_consensus.cc:1261] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.695442 6451 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.696718 3150 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.696954 5079 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.697320 6452 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.697883 6459 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.698673 3280 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:56.698974 3281 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.699594 6453 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.701400 3148 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.702375 5342 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.703281 6469 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.704097 6469 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.707937 3281 consensus_queue.cc:237] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:56.712302 6479 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.712935 5342 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.717115 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4cb5c6a514354708bff63dd5e51c7420 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.717020 3148 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.699565 6451 consensus_queue.cc:1035] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.722743 6453 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.728336 3538 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.728096 3410 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.732961 6035 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.733382 6035 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:56.733290 6479 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.733655 6035 ts_tablet_manager.cc:1367] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.153s user 0.038s sys 0.008s
I20240501 08:12:56.734247 3003 catalog_manager.cc:5484] T 6e2593a1a02d4f61972ba63e7dbc628c P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.734622 6035 raft_consensus.cc:348] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.734468 5474 raft_consensus.cc:2932] T 6e2593a1a02d4f61972ba63e7dbc628c P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:56.734850 6035 raft_consensus.cc:729] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:56.735049 6035 consensus_queue.cc:260] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:56.736191 6035 ts_tablet_manager.cc:1398] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:12:56.739354 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet 6e2593a1a02d4f61972ba63e7dbc628c with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:12:56.740473 3001 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:56.743297 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-f0c4a54ade9c4e96bb575bfbfa5d0206 received from {username='slave'} at 127.28.1.135:45681
I20240501 08:12:56.743427 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-f0c4a54ade9c4e96bb575bfbfa5d0206 on tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:56.745777 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 4cb5c6a514354708bff63dd5e51c7420 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:56.749545 5211 raft_consensus.cc:1261] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.750142 6479 consensus_queue.cc:1035] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.751726 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6e2593a1a02d4f61972ba63e7dbc628c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.752812 6479 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.757454 5210 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.763423 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7667242ae1c548ba982e74e190f2af27 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.763201 3539 raft_consensus.cc:2932] T 7667242ae1c548ba982e74e190f2af27 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.764935 6469 raft_consensus.cc:1050] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:12:56.765769 6469 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:56.766657 6500 tablet_replica.cc:330] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:12:56.767215 6500 raft_consensus.cc:2229] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.767592 6500 raft_consensus.cc:2258] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.768265 6502 tablet_replica.cc:330] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:12:56.768784 6502 raft_consensus.cc:2229] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.769104 6502 raft_consensus.cc:2258] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.767228 3001 catalog_manager.cc:5484] T 7667242ae1c548ba982e74e190f2af27 P 8151eaf288354d60b73b3c090dd271fd reported cstate change: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:56.770710 6502 ts_tablet_manager.cc:1875] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.771798 6500 ts_tablet_manager.cc:1875] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.772118 6459 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.774097 3538 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.775025 5391 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.775138 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet 7667242ae1c548ba982e74e190f2af27 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56990
I20240501 08:12:56.776906 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.777987 3280 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.778577 5391 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.779675 5342 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.780686 5391 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.781929 5474 raft_consensus.cc:1203] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.782230 3148 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.783327 5079 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.791465 3003 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.801764 6504 tablet_replica.cc:330] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:12:56.801643 3539 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:56.804701 3148 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.805006 6504 raft_consensus.cc:2229] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.805249 6453 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.805868 6504 raft_consensus.cc:2258] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.806277 5079 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.807675 6453 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.811569 3150 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.813982 6500 ts_tablet_manager.cc:1888] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.814110 6504 ts_tablet_manager.cc:1875] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.815003 6500 log.cc:1196] T 6e2593a1a02d4f61972ba63e7dbc628c P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/6e2593a1a02d4f61972ba63e7dbc628c
I20240501 08:12:56.815943 6452 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.816956 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet babe0d774b24482dab935eccb99f31c1 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.817212 3003 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.817658 5079 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.821404 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet 6e2593a1a02d4f61972ba63e7dbc628c (table rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]) successfully deleted
I20240501 08:12:56.825663 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet babe0d774b24482dab935eccb99f31c1 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:12:56.826989 6502 ts_tablet_manager.cc:1888] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.827143 6502 log.cc:1196] T 4cb5c6a514354708bff63dd5e51c7420 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/4cb5c6a514354708bff63dd5e51c7420
I20240501 08:12:56.828033 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 4cb5c6a514354708bff63dd5e51c7420 (table rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]) successfully deleted
I20240501 08:12:56.829016 6504 ts_tablet_manager.cc:1888] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.829170 6504 log.cc:1196] T 7667242ae1c548ba982e74e190f2af27 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/7667242ae1c548ba982e74e190f2af27
I20240501 08:12:56.830047 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet 7667242ae1c548ba982e74e190f2af27 (table rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]) successfully deleted
I20240501 08:12:56.831001 5211 raft_consensus.cc:1203] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.832330 6471 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.833001 5341 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.836846 3539 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.837435 3410 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.837879 3001 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.840317 6052 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.840837 6052 tablet_bootstrap.cc:492] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:12:56.841102 6052 ts_tablet_manager.cc:1367] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.244s user 0.040s sys 0.006s
I20240501 08:12:56.841905 6052 raft_consensus.cc:348] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.842154 6052 raft_consensus.cc:729] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:12:56.842344 6052 consensus_queue.cc:260] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:56.844530 6052 ts_tablet_manager.cc:1398] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:12:56.845216 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-fdeb6b683c7f4097ba3054ab8d4fb203 received from {username='slave'} at 127.28.1.134:35817
I20240501 08:12:56.845363 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-fdeb6b683c7f4097ba3054ab8d4fb203 on tablet fdeb6b683c7f4097ba3054ab8d4fb203 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:12:56.847440 3280 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:56.848616 6509 tablet_replica.cc:330] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:12:56.849504 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.849965 6471 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.850536 6509 raft_consensus.cc:2229] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.850958 6509 raft_consensus.cc:2258] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.851212 5341 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.853446 6479 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.853919 5391 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.854480 3410 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.859324 6509 ts_tablet_manager.cc:1875] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.859397 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7fde726fbe8d48f685a4580ff6ae4033 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.860770 5211 raft_consensus.cc:1203] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.863086 3005 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:56.865679 5342 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:56.867259 6509 ts_tablet_manager.cc:1888] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.867487 6509 log.cc:1196] T babe0d774b24482dab935eccb99f31c1 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/babe0d774b24482dab935eccb99f31c1
I20240501 08:12:56.867954 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 7fde726fbe8d48f685a4580ff6ae4033 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:56.868387 6502 tablet_replica.cc:330] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:12:56.868618 6502 raft_consensus.cc:2229] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.868844 6502 raft_consensus.cc:2258] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.868840 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet babe0d774b24482dab935eccb99f31c1 (table rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]) successfully deleted
I20240501 08:12:56.869837 6502 ts_tablet_manager.cc:1875] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.876286 6502 ts_tablet_manager.cc:1888] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.876399 6502 log.cc:1196] T 7fde726fbe8d48f685a4580ff6ae4033 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:12:56.877100 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 7fde726fbe8d48f685a4580ff6ae4033 (table rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]) successfully deleted
I20240501 08:12:56.878253 5474 raft_consensus.cc:1203] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:56.892532 6452 raft_consensus.cc:1050] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:12:56.893363 6452 consensus_queue.cc:237] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:56.895650 5079 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.896224 3150 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.896298 6451 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.896797 3280 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.897352 6451 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.897658 6453 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.900916 6451 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.901504 3280 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.901825 3150 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.904680 5079 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.906229 3001 catalog_manager.cc:5484] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:56.912408 3538 consensus_queue.cc:237] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:56.914475 5079 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.914813 3150 raft_consensus.cc:1261] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:56.915211 6453 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.915457 6452 consensus_queue.cc:1035] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:56.917881 5726 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.918531 5079 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.920208 6087 tablet_copy_client.cc:523] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.920540 3150 raft_consensus.cc:2932] T 0612f1f8e4ca4a29971151bf52571a33 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.921908 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0612f1f8e4ca4a29971151bf52571a33 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:56.923395 6087 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:56.923698 3005 catalog_manager.cc:5484] T 0612f1f8e4ca4a29971151bf52571a33 P f0af5c767c1b496cb1163870b60b066a reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:56.927708 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 0612f1f8e4ca4a29971151bf52571a33 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:12:56.928076 6509 tablet_replica.cc:330] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:12:56.928288 6509 raft_consensus.cc:2229] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:56.928627 6509 raft_consensus.cc:2258] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:56.929723 6509 ts_tablet_manager.cc:1875] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:56.935637 6509 ts_tablet_manager.cc:1888] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:56.936026 6509 log.cc:1196] T 0612f1f8e4ca4a29971151bf52571a33 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/0612f1f8e4ca4a29971151bf52571a33
I20240501 08:12:56.936869 6090 tablet_copy_client.cc:523] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.937214 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 0612f1f8e4ca4a29971151bf52571a33 (table rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]) successfully deleted
I20240501 08:12:56.940590 6090 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:56.949975 6075 tablet_copy_client.cc:523] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:56.953610 6075 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:56.964267 6087 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:56.964709 6087 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:56.965023 6087 ts_tablet_manager.cc:1367] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.042s user 0.037s sys 0.005s
I20240501 08:12:56.965886 6087 raft_consensus.cc:348] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.966173 6087 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:56.966364 6087 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:56.967478 6087 ts_tablet_manager.cc:1398] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:12:56.968425 3561 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-d56791555f6644a3bcee602299a85761 received from {username='slave'} at 127.28.1.133:56329
I20240501 08:12:56.968585 3561 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-d56791555f6644a3bcee602299a85761 on tablet d56791555f6644a3bcee602299a85761 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:56.983523 6537 raft_consensus.cc:1050] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:12:56.984428 6537 consensus_queue.cc:237] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:56.987763 5211 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.988418 3280 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.988438 6541 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.989018 3410 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:56.989826 6542 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:56.998576 6541 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.007331 5079 raft_consensus.cc:1203] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:57.024984 6471 raft_consensus.cc:1050] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:12:57.025779 6471 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.025967 6542 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.028777 5210 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.035279 5474 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.035566 3538 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.036023 6469 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.036789 3005 catalog_manager.cc:5484] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd reported cstate change: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.037335 6469 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.039502 3280 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.041813 3410 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.043468 3150 consensus_queue.cc:237] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:57.045513 5211 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.046332 6537 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.048523 3150 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.049115 6469 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.049582 6537 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.053977 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ea2ecf61668b488c96b903993ab07b16 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.055497 5210 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.057246 3005 catalog_manager.cc:5484] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.061136 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet ea2ecf61668b488c96b903993ab07b16 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56990
I20240501 08:12:57.063541 3279 raft_consensus.cc:1261] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.064496 6541 consensus_queue.cc:1035] T ea2ecf61668b488c96b903993ab07b16 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.064375 5391 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.064900 5079 raft_consensus.cc:1203] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:57.066082 3279 raft_consensus.cc:2932] T ea2ecf61668b488c96b903993ab07b16 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.069080 5474 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.071062 3538 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.074821 6504 tablet_replica.cc:330] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:12:57.075095 6504 raft_consensus.cc:2229] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.075323 6504 raft_consensus.cc:2258] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.078272 6504 ts_tablet_manager.cc:1875] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.083207 6504 ts_tablet_manager.cc:1888] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.083319 6504 log.cc:1196] T ea2ecf61668b488c96b903993ab07b16 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/ea2ecf61668b488c96b903993ab07b16
I20240501 08:12:57.084035 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet ea2ecf61668b488c96b903993ab07b16 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:12:57.086025 3150 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.088716 3001 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.094600 3279 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.100261 5473 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.101763 5747 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.103610 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet ea2ecf61668b488c96b903993ab07b16 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 5 (current committed config index is 6)) from {username='slave'} at 127.0.0.1:56990
I20240501 08:12:57.104213 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet ea2ecf61668b488c96b903993ab07b16 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:12:57.105403 5747 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.106048 3539 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.106848 6471 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.108397 3539 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.109426 5474 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.113543 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d749fd4842294f8fba36f287b7072c9d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.114512 3005 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.118434 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet d749fd4842294f8fba36f287b7072c9d with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:12:57.118902 6500 tablet_replica.cc:330] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:12:57.119129 6500 raft_consensus.cc:2229] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.119421 6500 raft_consensus.cc:2258] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.120363 6500 ts_tablet_manager.cc:1875] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.127012 6500 ts_tablet_manager.cc:1888] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.127207 6500 log.cc:1196] T d749fd4842294f8fba36f287b7072c9d P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/d749fd4842294f8fba36f287b7072c9d
I20240501 08:12:57.129038 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet d749fd4842294f8fba36f287b7072c9d (table rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]) successfully deleted
I20240501 08:12:57.129173 6090 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:57.133378 6090 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:57.133738 6090 ts_tablet_manager.cc:1367] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.193s user 0.053s sys 0.002s
I20240501 08:12:57.134634 6090 raft_consensus.cc:348] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:57.134902 6090 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:57.135108 6090 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:57.135737 6090 ts_tablet_manager.cc:1398] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:12:57.145820 6075 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:57.145788 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-885de7b67f474f7bb54badfae193906d received from {username='slave'} at 127.28.1.136:60049
I20240501 08:12:57.145947 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-885de7b67f474f7bb54badfae193906d on tablet 885de7b67f474f7bb54badfae193906d with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:57.146281 6075 tablet_bootstrap.cc:492] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:57.146930 6075 ts_tablet_manager.cc:1367] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.193s user 0.056s sys 0.007s
I20240501 08:12:57.148097 6075 raft_consensus.cc:348] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:57.148398 6075 raft_consensus.cc:729] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:57.148458 6471 raft_consensus.cc:1050] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:12:57.148600 6075 consensus_queue.cc:260] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:57.149263 6471 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.151618 6075 ts_tablet_manager.cc:1398] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:12:57.152329 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-d3db152c64e74ee3a788769dba03b7a6 received from {username='slave'} at 127.28.1.135:40907
I20240501 08:12:57.152487 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-d3db152c64e74ee3a788769dba03b7a6 on tablet d3db152c64e74ee3a788769dba03b7a6 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:57.158856 3538 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.159078 3410 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.159258 5079 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.159423 6472 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.159838 5747 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.160210 5391 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.162971 5747 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.164597 5079 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.168113 3538 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.172183 3001 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.178665 5726 raft_consensus.cc:1050] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:12:57.179526 5726 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:57.180029 3411 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.191026 3279 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.191780 5342 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.192793 5726 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.193006 3279 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.193193 3410 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.193482 3411 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.193449 6453 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.193766 6472 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.193991 5079 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.194161 6453 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.194732 6472 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.197775 6479 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.198022 6458 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.198273 5079 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.198580 5342 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.202536 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 460948db7dfd4d5589d478ddd5c6c10c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.202852 3411 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.203743 3000 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.205121 3279 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.209682 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 460948db7dfd4d5589d478ddd5c6c10c with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:57.210031 6502 tablet_replica.cc:330] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:12:57.210233 6502 raft_consensus.cc:2229] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.210592 6502 raft_consensus.cc:2258] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.210489 3005 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.212842 6502 ts_tablet_manager.cc:1875] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.219892 6502 ts_tablet_manager.cc:1888] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.220039 6502 log.cc:1196] T 460948db7dfd4d5589d478ddd5c6c10c P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:12:57.220671 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 460948db7dfd4d5589d478ddd5c6c10c (table rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]) successfully deleted
I20240501 08:12:57.222564 3410 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.224731 3538 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:57.226676 3410 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.227141 6458 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.227524 5342 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.228035 6458 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.229187 6452 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.229714 3410 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.231827 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e21419bf69e746baa7a96fbf5eee384d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.233842 5342 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.234197 3000 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.237589 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet e21419bf69e746baa7a96fbf5eee384d with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:12:57.237936 6509 tablet_replica.cc:330] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:12:57.238137 6509 raft_consensus.cc:2229] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.238406 6509 raft_consensus.cc:2258] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.239251 6509 ts_tablet_manager.cc:1875] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.243609 6509 ts_tablet_manager.cc:1888] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.243707 6509 log.cc:1196] T e21419bf69e746baa7a96fbf5eee384d P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/e21419bf69e746baa7a96fbf5eee384d
I20240501 08:12:57.244330 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet e21419bf69e746baa7a96fbf5eee384d (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:12:57.260959 6542 raft_consensus.cc:1050] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:12:57.261621 6542 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.264034 5474 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.264060 3279 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.264571 3410 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.264513 6541 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.264780 5532 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.265091 6542 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.267838 6542 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.268443 3279 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.269358 3410 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.271479 5474 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.272900 3005 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.277925 3150 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.279458 5474 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.279758 3279 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.279907 5532 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.280434 6542 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.281920 5532 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.282474 5474 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.284979 3279 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.285392 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a3fce8868ca74a028492122fc2762001 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.287359 3001 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.290738 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet a3fce8868ca74a028492122fc2762001 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56990
I20240501 08:12:57.291142 6504 tablet_replica.cc:330] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:12:57.291363 6504 raft_consensus.cc:2229] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.291712 6504 raft_consensus.cc:2258] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.292595 6504 ts_tablet_manager.cc:1875] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.297013 5342 raft_consensus.cc:1203] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:57.300045 6504 ts_tablet_manager.cc:1888] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.300208 6504 log.cc:1196] T a3fce8868ca74a028492122fc2762001 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/a3fce8868ca74a028492122fc2762001
I20240501 08:12:57.300969 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet a3fce8868ca74a028492122fc2762001 (table rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]) successfully deleted
I20240501 08:12:57.301230 6570 raft_consensus.cc:1050] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:12:57.301949 6570 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.304265 3279 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.304539 3150 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.304636 5474 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.304837 5908 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.305150 6570 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.305440 6572 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.309257 6570 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.310037 3150 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.315183 5474 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.315582 3000 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.322206 3279 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.349421 6412 heartbeater.cc:503] Master 127.28.1.190:38185 was elected leader, sending a full tablet report...
I20240501 08:12:57.357362 5474 raft_consensus.cc:1203] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:57.395033 6459 raft_consensus.cc:1050] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:12:57.395838 6459 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:57.398792 3279 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.399662 6452 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.400224 5210 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.400378 3150 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.401588 5726 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.402127 5726 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.404428 6451 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.404999 3148 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.407320 5210 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.408802 3279 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.410476 3001 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.411412 6541 raft_consensus.cc:1050] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:12:57.412271 6541 consensus_queue.cc:237] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:57.415956 3538 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:57.418272 5210 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.418974 6451 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.419217 3410 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.419585 5211 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.419883 6541 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.420522 3538 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.420542 3150 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.420619 5532 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.421391 5532 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.421898 6451 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.422575 6452 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.425072 3150 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.431073 3005 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.431563 5210 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.434150 6542 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.435663 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet a68f4cc374ad4656b1de788033ece98c with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:12:57.436017 6509 tablet_replica.cc:330] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:12:57.436221 6509 raft_consensus.cc:2229] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.436451 6509 raft_consensus.cc:2258] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.436971 3538 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.438113 5210 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.438386 3410 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.439450 3005 catalog_manager.cc:5484] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.440586 6509 ts_tablet_manager.cc:1875] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.443897 3150 consensus_queue.cc:237] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:12:57.445574 5210 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.445987 3410 raft_consensus.cc:1261] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.446099 6542 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.446235 6509 ts_tablet_manager.cc:1888] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.446345 6509 log.cc:1196] T a68f4cc374ad4656b1de788033ece98c P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/a68f4cc374ad4656b1de788033ece98c
I20240501 08:12:57.446633 5532 consensus_queue.cc:1035] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.449330 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a68f4cc374ad4656b1de788033ece98c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.449162 6537 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.449772 5209 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.452111 6458 raft_consensus.cc:1050] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:12:57.452847 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fdeb6b683c7f4097ba3054ab8d4fb203 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.452929 6458 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.453176 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet a68f4cc374ad4656b1de788033ece98c (table rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]) successfully deleted
I20240501 08:12:57.455016 3000 catalog_manager.cc:5484] T fdeb6b683c7f4097ba3054ab8d4fb203 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.456494 3411 raft_consensus.cc:2932] T fdeb6b683c7f4097ba3054ab8d4fb203 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:12:57.460786 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet fdeb6b683c7f4097ba3054ab8d4fb203 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:57.461146 6502 tablet_replica.cc:330] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:12:57.461371 6502 raft_consensus.cc:2229] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.461658 6502 raft_consensus.cc:2258] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.463562 6502 ts_tablet_manager.cc:1875] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.470410 6502 ts_tablet_manager.cc:1888] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.470598 6502 log.cc:1196] T fdeb6b683c7f4097ba3054ab8d4fb203 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/fdeb6b683c7f4097ba3054ab8d4fb203
I20240501 08:12:57.471385 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet fdeb6b683c7f4097ba3054ab8d4fb203 (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:12:57.477424 5079 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.477900 3148 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.477957 3410 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.478014 5726 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.478431 6452 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.478953 5726 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.489173 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet fdeb6b683c7f4097ba3054ab8d4fb203 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 5 (current committed config index is 6)) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:57.489909 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet fdeb6b683c7f4097ba3054ab8d4fb203 (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:12:57.491019 5726 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.491863 3150 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.494606 3411 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.497628 3001 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.498793 5079 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.508486 3538 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.510286 5079 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.510731 3411 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.511236 6452 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.511547 5726 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.513756 6451 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.514544 5079 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.516280 3411 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.517810 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d56791555f6644a3bcee602299a85761 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.519994 3003 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.526124 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet d56791555f6644a3bcee602299a85761 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:12:57.526499 6500 tablet_replica.cc:330] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:12:57.526696 6500 raft_consensus.cc:2229] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.526926 6500 raft_consensus.cc:2258] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.528563 6500 ts_tablet_manager.cc:1875] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.535117 6500 ts_tablet_manager.cc:1888] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.535334 6500 log.cc:1196] T d56791555f6644a3bcee602299a85761 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/d56791555f6644a3bcee602299a85761
I20240501 08:12:57.536121 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet d56791555f6644a3bcee602299a85761 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:12:57.554291 3411 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa"
dest_uuid: "254a7288f105497eb60d1709036a71a9"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38752
I20240501 08:12:57.554540 3411 raft_consensus.cc:595] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:12:57.583832 5342 raft_consensus.cc:1203] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:57.620967 6479 raft_consensus.cc:1050] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:12:57.621816 6479 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.624437 3150 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.624457 3538 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.625065 6479 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.625594 5079 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.626106 6479 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.626940 6479 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.629626 6471 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.633417 5079 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.638296 3001 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.640830 3150 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.643013 3279 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:12:57.645321 5079 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.645850 6479 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.646175 3148 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.646865 6479 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.649545 5747 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.650223 3148 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.653585 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d110b6fb2855454892ab5359ec3210b0 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.655381 5079 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:12:57.657362 3001 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.662503 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet d110b6fb2855454892ab5359ec3210b0 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:12:57.662925 6502 tablet_replica.cc:330] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:12:57.663164 6502 raft_consensus.cc:2229] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.663349 6502 pending_rounds.cc:62] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Trying to abort 1 pending ops.
I20240501 08:12:57.663458 6502 pending_rounds.cc:68] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7022801623536758784 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "d110b6fb2855454892ab5359ec3210b0" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } } }
I20240501 08:12:57.663915 6502 raft_consensus.cc:2894] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240501 08:12:57.664076 6502 raft_consensus.cc:2866] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240501 08:12:57.664188 6502 raft_consensus.cc:2258] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.666441 6502 ts_tablet_manager.cc:1875] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.671377 6502 ts_tablet_manager.cc:1888] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.671478 6502 log.cc:1196] T d110b6fb2855454892ab5359ec3210b0 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/d110b6fb2855454892ab5359ec3210b0
I20240501 08:12:57.672158 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet d110b6fb2855454892ab5359ec3210b0 (table rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]) successfully deleted
I20240501 08:12:57.720352 3411 consensus_queue.cc:237] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:57.722532 3148 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.722487 3538 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:57.722812 3345 consensus_peers.cc:485] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 901e7e9545d94ee2904a29c5675ccc75. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.723065 5906 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.723366 6572 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.725804 5908 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.726395 3538 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.726707 3148 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.731031 3005 catalog_manager.cc:5484] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.746302 6471 raft_consensus.cc:1050] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:12:57.747108 6471 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:57.749226 3411 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.749214 5342 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.749539 3148 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.749768 6479 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.750190 6472 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.750459 6471 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.752434 3538 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:57.753445 6471 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.754053 3148 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.758054 3411 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.759958 3001 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.762547 3279 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:57.762619 3475 consensus_peers.cc:485] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0434084d13f740368d1daae53dee9fe2. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.763077 3148 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.763517 6451 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.764189 6453 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.770421 5342 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.772907 3281 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:57.774891 3411 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.775741 6471 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.777488 6570 raft_consensus.cc:1050] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:12:57.778275 6570 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.778544 5342 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.779104 6471 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.780329 6453 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.781817 3411 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.782209 6472 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.786283 5341 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.786732 5474 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.787894 5906 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.788734 3281 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.791314 3005 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:57.793738 3150 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.794626 6572 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.794941 3281 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:57.795419 6572 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:57.796486 6572 raft_consensus.cc:979] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: : Instructing follower e3b81e5e25de4eddbeaad1e5834c0836 to start an election
I20240501 08:12:57.796624 6572 raft_consensus.cc:1067] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Signalling peer e3b81e5e25de4eddbeaad1e5834c0836 to start an election
I20240501 08:12:57.793443 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.798050 3001 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.800906 3148 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.801313 3279 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa"
dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
from {username='slave'} at 127.28.1.131:38603
I20240501 08:12:57.801577 3279 raft_consensus.cc:482] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:12:57.801708 3279 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:57.802182 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:12:57.802522 6500 tablet_replica.cc:330] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:12:57.802714 6500 raft_consensus.cc:2229] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.803071 6500 raft_consensus.cc:2258] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.804051 6500 ts_tablet_manager.cc:1875] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.804392 3279 raft_consensus.cc:504] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.806715 3279 leader_election.cc:288] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 2 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523)
I20240501 08:12:57.806980 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:12:57.807235 3411 raft_consensus.cc:3032] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:12:57.807349 3411 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:12:57.807694 3411 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.808154 3411 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:57.808956 5473 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
I20240501 08:12:57.809288 5473 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:57.809577 3148 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c927478b05c14f47b736faf52b4db9aa" candidate_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:12:57.809792 3148 raft_consensus.cc:3037] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:12:57.810917 3411 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 2.
I20240501 08:12:57.811933 6500 ts_tablet_manager.cc:1888] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.812081 6500 log.cc:1196] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:12:57.812393 5473 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 2.
I20240501 08:12:57.812536 3148 raft_consensus.cc:2454] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e3b81e5e25de4eddbeaad1e5834c0836 in term 2.
I20240501 08:12:57.812983 3216 leader_election.cc:302] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:12:57.813057 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet f0c4a54ade9c4e96bb575bfbfa5d0206 (table rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]) successfully deleted
I20240501 08:12:57.813336 6471 raft_consensus.cc:2781] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:12:57.813490 6471 raft_consensus.cc:686] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 LEADER]: Becoming Leader. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:12:57.813794 6471 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.817852 3003 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: term changed from 1 to 2, leader changed from 254a7288f105497eb60d1709036a71a9 (127.28.1.131) to e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130). New cstate: current_term: 2 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.817852 6570 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.819031 3150 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.822894 5473 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.823616 3003 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:57.827188 3280 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.831588 3411 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:57.834013 3150 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.834297 5473 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:57.834525 6570 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.834934 5906 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:57.837111 5906 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.838232 5473 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.838238 3150 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.841059 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 885de7b67f474f7bb54badfae193906d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:57.843374 3001 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:57.847034 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 885de7b67f474f7bb54badfae193906d with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:12:57.847596 6509 tablet_replica.cc:330] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:12:57.847786 6509 raft_consensus.cc:2229] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:57.848141 6509 raft_consensus.cc:2258] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:57.849512 6509 ts_tablet_manager.cc:1875] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:57.851246 3280 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
W20240501 08:12:57.854032 3216 consensus_peers.cc:485] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bdad8a9a16414c5698f9fafb7272bdb1. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.854180 3538 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.854254 3411 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.854754 6472 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.855047 6471 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.855826 6509 ts_tablet_manager.cc:1888] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:57.855990 6509 log.cc:1196] T 885de7b67f474f7bb54badfae193906d P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/885de7b67f474f7bb54badfae193906d
I20240501 08:12:57.856716 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 885de7b67f474f7bb54badfae193906d (table rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]) successfully deleted
I20240501 08:12:57.858399 6472 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:57.859031 3538 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:57.862052 3411 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:12:57.863842 3000 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.877012 3411 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:57.879992 3280 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.880257 3538 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.880635 5908 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.881095 6570 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.883945 5908 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
W20240501 08:12:57.886407 3344 consensus_peers.cc:485] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c3672fbf92d4b54a91554f88382a5ae. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.886755 3280 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:57.887636 3538 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:57.889351 3003 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.903571 3280 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:57.905793 3411 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.905810 3538 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:57.906277 3215 consensus_peers.cc:485] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f75a6080c937423f8ffb9f75b96da670. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.906399 6472 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.906700 6479 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.909298 6471 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.909910 3411 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.910043 3538 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.914203 3001 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.928803 3538 consensus_queue.cc:237] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:57.930871 3280 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.931216 3411 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.931433 6451 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:57.931718 3475 consensus_peers.cc:485] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 36c9ad94ddc54bc6ab5021a0d297a774. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.932174 6451 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.934734 5726 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.935549 3411 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.938707 3280 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:57.942939 3003 catalog_manager.cc:5484] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.953485 3411 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:57.955674 3280 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.955822 3150 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:57.956035 3344 consensus_peers.cc:485] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4adf393d330f45ba95fb50c850f20348. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.956287 5908 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.956561 6570 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.959167 5906 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:57.960057 3150 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:57.960497 3280 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:57.965149 3005 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:57.980386 3150 consensus_queue.cc:237] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:57.982689 3538 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:57.982709 3411 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:57.983098 3084 consensus_peers.cc:485] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 313e1bed969b4a4ba124116ba5aed741. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:57.983228 5532 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.983528 6542 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:57.986114 6541 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.986713 3538 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.986790 3411 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:57.992084 3003 catalog_manager.cc:5484] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:58.005172 3150 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:58.007584 3280 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:58.007772 3411 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:58.008314 5532 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:58.008726 6541 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:58.012116 6542 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:58.013011 3280 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:58.019194 3005 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:58.026268 5906 raft_consensus.cc:1050] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:12:58.027220 5906 consensus_queue.cc:237] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:58.032091 3410 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:12:58.038627 5342 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240501 08:12:58.038749 3084 consensus_peers.cc:485] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e62d72ab5f9b460aaa7fe7bb7ecd7b08. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:58.039427 3150 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:58.039448 6572 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:58.040035 5908 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:58.047083 3538 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:12:58.051599 5908 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:12:58.053259 6572 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.056823 3148 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.057771 5342 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.058128 3005 catalog_manager.cc:5484] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:58.071067 3539 consensus_queue.cc:237] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:58.076274 3410 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:58.076961 3150 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:58.076360 3539 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.077566 6451 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240501 08:12:58.079020 3474 consensus_peers.cc:485] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0bd2ea6b1e714d619d7dc00d20067a59. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:58.079278 6451 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:58.080118 3410 consensus_queue.cc:237] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:12:58.082340 5342 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:58.083068 6570 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:58.085505 3539 raft_consensus.cc:1261] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:12:58.086426 6570 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.088429 5906 consensus_queue.cc:1035] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:58.089962 3538 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.092916 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d3db152c64e74ee3a788769dba03b7a6 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:58.094190 5342 raft_consensus.cc:2932] T d3db152c64e74ee3a788769dba03b7a6 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:12:58.094517 6451 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:58.102893 3003 catalog_manager.cc:5484] T d3db152c64e74ee3a788769dba03b7a6 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:58.101327 3149 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:58.109310 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet d3db152c64e74ee3a788769dba03b7a6 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:12:58.109890 6500 tablet_replica.cc:330] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:12:58.110046 6500 raft_consensus.cc:2229] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:58.110386 3003 catalog_manager.cc:5484] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:58.112336 6500 raft_consensus.cc:2258] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:58.113368 3411 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:12:58.114230 6500 ts_tablet_manager.cc:1875] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:58.120523 6500 ts_tablet_manager.cc:1888] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:12:58.120672 6500 log.cc:1196] T d3db152c64e74ee3a788769dba03b7a6 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/d3db152c64e74ee3a788769dba03b7a6
I20240501 08:12:58.121412 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet d3db152c64e74ee3a788769dba03b7a6 (table rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]) successfully deleted
I20240501 08:12:58.220160 6829 ts_tablet_manager.cc:902] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:58.229437 6829 tablet_copy_client.cc:308] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:58.230175 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 0434084d13f740368d1daae53dee9fe2 from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:60309)
I20240501 08:12:58.230352 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 0434084d13f740368d1daae53dee9fe2 from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:60309: session id = b1a5f0e8d89547a1ac983c6a5256eca8-0434084d13f740368d1daae53dee9fe2
I20240501 08:12:58.233106 3561 tablet_copy_source_session.cc:205] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.234246 6829 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0434084d13f740368d1daae53dee9fe2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.239720 6829 tablet_copy_client.cc:783] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.239998 6829 tablet_copy_client.cc:647] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.241645 6838 tablet_copy_client.cc:1002] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.254854 3149 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:12:58.255409 6479 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:58.258293 3410 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:12:58.259173 6479 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:58.266759 5474 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:12:58.269860 5747 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:12:58.276368 6863 ts_tablet_manager.cc:902] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:58.281035 6863 tablet_copy_client.cc:308] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:58.281807 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 901e7e9545d94ee2904a29c5675ccc75 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:40907)
I20240501 08:12:58.281999 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 901e7e9545d94ee2904a29c5675ccc75 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:40907: session id = 3e6a44464d7748fa8c784ef795b896fb-901e7e9545d94ee2904a29c5675ccc75
I20240501 08:12:58.284440 3430 tablet_copy_source_session.cc:205] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.285749 6863 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 901e7e9545d94ee2904a29c5675ccc75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.291159 6863 tablet_copy_client.cc:783] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.291414 6863 tablet_copy_client.cc:647] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.292999 6869 tablet_copy_client.cc:1002] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.300541 6878 ts_tablet_manager.cc:902] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:58.301766 6878 tablet_copy_client.cc:308] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:58.302526 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet bdad8a9a16414c5698f9fafb7272bdb1 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:12:58.302706 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet bdad8a9a16414c5698f9fafb7272bdb1 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:12:58.305526 3301 tablet_copy_source_session.cc:205] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.306948 6878 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bdad8a9a16414c5698f9fafb7272bdb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.313139 6878 tablet_copy_client.cc:783] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.313526 6878 tablet_copy_client.cc:647] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.315790 6882 tablet_copy_client.cc:1002] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.323522 3280 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:58.325726 3149 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:12:58.325709 3539 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:12:58.326012 3215 consensus_peers.cc:485] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6da0f7473ef248f2ad4f5621f3fbd25e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:12:58.326260 6471 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:58.326567 5747 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:12:58.329165 5391 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:58.329747 3149 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:58.332439 3539 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:12:58.334306 3003 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index -1 to 4, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:12:58.340943 6898 ts_tablet_manager.cc:902] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:58.342459 6898 tablet_copy_client.cc:308] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:58.348497 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 4adf393d330f45ba95fb50c850f20348 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:33215)
I20240501 08:12:58.348686 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 4adf393d330f45ba95fb50c850f20348 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:33215: session id = f0af5c767c1b496cb1163870b60b066a-4adf393d330f45ba95fb50c850f20348
I20240501 08:12:58.350675 3430 tablet_copy_source_session.cc:205] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.352370 6898 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4adf393d330f45ba95fb50c850f20348. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.357779 6898 tablet_copy_client.cc:783] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.358095 6898 tablet_copy_client.cc:647] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.359915 6902 tablet_copy_client.cc:1002] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.379400 6922 ts_tablet_manager.cc:902] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:58.386919 6922 tablet_copy_client.cc:308] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:12:58.387742 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet 313e1bed969b4a4ba124116ba5aed741 from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:35605)
I20240501 08:12:58.387926 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet 313e1bed969b4a4ba124116ba5aed741 from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:35605: session id = b1a5f0e8d89547a1ac983c6a5256eca8-313e1bed969b4a4ba124116ba5aed741
I20240501 08:12:58.390492 3170 tablet_copy_source_session.cc:205] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.392632 6922 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313e1bed969b4a4ba124116ba5aed741. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.398234 6922 tablet_copy_client.cc:783] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.398473 6922 tablet_copy_client.cc:647] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.401762 6926 tablet_copy_client.cc:1002] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.402251 6937 ts_tablet_manager.cc:902] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:58.404517 6937 tablet_copy_client.cc:308] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:58.405196 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet f75a6080c937423f8ffb9f75b96da670 from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:42911)
I20240501 08:12:58.405397 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet f75a6080c937423f8ffb9f75b96da670 from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:42911: session id = b1a5f0e8d89547a1ac983c6a5256eca8-f75a6080c937423f8ffb9f75b96da670
I20240501 08:12:58.407575 3301 tablet_copy_source_session.cc:205] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.408802 6937 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75a6080c937423f8ffb9f75b96da670. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.415254 6937 tablet_copy_client.cc:783] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.415529 6937 tablet_copy_client.cc:647] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.417568 6941 tablet_copy_client.cc:1002] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.445964 6962 ts_tablet_manager.cc:902] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:12:58.447319 6962 tablet_copy_client.cc:308] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:12:58.447997 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 1c3672fbf92d4b54a91554f88382a5ae from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:33215)
I20240501 08:12:58.448163 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 1c3672fbf92d4b54a91554f88382a5ae from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:33215: session id = f0af5c767c1b496cb1163870b60b066a-1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:12:58.450374 3430 tablet_copy_source_session.cc:205] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.452126 6962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c3672fbf92d4b54a91554f88382a5ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.456881 6962 tablet_copy_client.cc:783] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.457110 6962 tablet_copy_client.cc:647] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.458591 6966 tablet_copy_client.cc:1002] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.473500 6977 ts_tablet_manager.cc:902] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:58.482465 6977 tablet_copy_client.cc:308] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:58.483197 6981 ts_tablet_manager.cc:902] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:12:58.483237 3561 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 36c9ad94ddc54bc6ab5021a0d297a774 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:44357)
I20240501 08:12:58.483404 3561 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 36c9ad94ddc54bc6ab5021a0d297a774 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:44357: session id = 3e6a44464d7748fa8c784ef795b896fb-36c9ad94ddc54bc6ab5021a0d297a774
I20240501 08:12:58.485028 6981 tablet_copy_client.cc:308] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:12:58.485743 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 0bd2ea6b1e714d619d7dc00d20067a59 from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:56329)
I20240501 08:12:58.485901 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 0bd2ea6b1e714d619d7dc00d20067a59 from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:56329: session id = f0af5c767c1b496cb1163870b60b066a-0bd2ea6b1e714d619d7dc00d20067a59
I20240501 08:12:58.486260 3561 tablet_copy_source_session.cc:205] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.487577 6977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36c9ad94ddc54bc6ab5021a0d297a774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.488413 3560 tablet_copy_source_session.cc:205] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.492153 6977 tablet_copy_client.cc:783] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.492393 6977 tablet_copy_client.cc:647] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.493548 6981 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bd2ea6b1e714d619d7dc00d20067a59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.493963 6982 tablet_copy_client.cc:1002] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.499665 6981 tablet_copy_client.cc:783] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.500527 6981 tablet_copy_client.cc:647] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.501943 6986 tablet_copy_client.cc:1002] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.598016 6987 ts_tablet_manager.cc:902] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:12:58.599637 6987 tablet_copy_client.cc:308] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:12:58.608911 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:60045)
I20240501 08:12:58.609082 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:60045: session id = 3e6a44464d7748fa8c784ef795b896fb-e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:12:58.611264 3170 tablet_copy_source_session.cc:205] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.612509 6987 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.617595 6987 tablet_copy_client.cc:783] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.617843 6987 tablet_copy_client.cc:647] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.619235 6991 tablet_copy_client.cc:1002] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:58.722438 3280 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:12:58.724620 5474 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240501 08:12:58.726174 3149 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240501 08:12:58.728343 6472 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:12:58.728907 6472 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:12:58.730788 6471 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:58.733309 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c927478b05c14f47b736faf52b4db9aa with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:12:58.733927 5474 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:58.736033 3000 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 2 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:12:58.737044 3149 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:12:58.741449 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet c927478b05c14f47b736faf52b4db9aa with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:56990
I20240501 08:12:58.750865 7016 tablet_replica.cc:330] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:12:58.751168 7016 raft_consensus.cc:2229] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:12:58.751369 7016 raft_consensus.cc:2258] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:12:58.752198 7016 ts_tablet_manager.cc:1875] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:12:58.756583 7016 ts_tablet_manager.cc:1888] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240501 08:12:58.756706 7016 log.cc:1196] T c927478b05c14f47b736faf52b4db9aa P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/c927478b05c14f47b736faf52b4db9aa
I20240501 08:12:58.757681 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet c927478b05c14f47b736faf52b4db9aa (table rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]) successfully deleted
I20240501 08:12:58.785553 7032 ts_tablet_manager.cc:902] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:12:58.790405 7032 tablet_copy_client.cc:308] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:12:58.791347 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 6da0f7473ef248f2ad4f5621f3fbd25e from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:42911)
I20240501 08:12:58.791486 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 6da0f7473ef248f2ad4f5621f3fbd25e from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:42911: session id = b1a5f0e8d89547a1ac983c6a5256eca8-6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:12:58.793725 3301 tablet_copy_source_session.cc:205] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:12:58.794878 7032 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6da0f7473ef248f2ad4f5621f3fbd25e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:12:58.800549 7032 tablet_copy_client.cc:783] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:12:58.800776 7032 tablet_copy_client.cc:647] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:12:58.802397 7038 tablet_copy_client.cc:1002] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:12:59.742023 6829 tablet_copy_client.cc:523] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.744966 6829 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:59.776551 6829 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.776917 6829 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:59.777225 6829 ts_tablet_manager.cc:1367] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.032s user 0.023s sys 0.009s
I20240501 08:12:59.777894 6829 raft_consensus.cc:348] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.778100 6829 raft_consensus.cc:729] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:59.778326 6829 consensus_queue.cc:260] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.779091 6829 ts_tablet_manager.cc:1398] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:12:59.779986 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-0434084d13f740368d1daae53dee9fe2 received from {username='slave'} at 127.28.1.136:60309
I20240501 08:12:59.780140 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-0434084d13f740368d1daae53dee9fe2 on tablet 0434084d13f740368d1daae53dee9fe2 with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:59.794026 6863 tablet_copy_client.cc:523] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.797361 6863 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:59.816219 6878 tablet_copy_client.cc:523] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.819593 6878 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:12:59.847344 5474 raft_consensus.cc:1203] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:59.857113 6863 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.857635 6863 tablet_bootstrap.cc:492] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:12:59.857918 6863 ts_tablet_manager.cc:1367] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.061s user 0.038s sys 0.010s
I20240501 08:12:59.858860 6863 raft_consensus.cc:348] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:59.859133 6863 raft_consensus.cc:729] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:12:59.859339 6863 consensus_queue.cc:260] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:12:59.860328 6898 tablet_copy_client.cc:523] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.864804 6898 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:59.874887 6863 ts_tablet_manager.cc:1398] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.017s user 0.003s sys 0.013s
I20240501 08:12:59.876714 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-901e7e9545d94ee2904a29c5675ccc75 received from {username='slave'} at 127.28.1.135:40907
I20240501 08:12:59.876852 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-901e7e9545d94ee2904a29c5675ccc75 on tablet 901e7e9545d94ee2904a29c5675ccc75 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:12:59.902227 6922 tablet_copy_client.cc:523] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.902840 6878 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.903245 6878 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:12:59.903540 6878 ts_tablet_manager.cc:1367] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.084s user 0.047s sys 0.012s
I20240501 08:12:59.904405 6878 raft_consensus.cc:348] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:59.904676 6878 raft_consensus.cc:729] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:12:59.904899 6878 consensus_queue.cc:260] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:12:59.906168 6922 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:59.912537 6878 ts_tablet_manager.cc:1398] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.009s user 0.007s sys 0.004s
I20240501 08:12:59.913672 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-bdad8a9a16414c5698f9fafb7272bdb1 received from {username='slave'} at 127.28.1.134:50227
I20240501 08:12:59.913856 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-bdad8a9a16414c5698f9fafb7272bdb1 on tablet bdad8a9a16414c5698f9fafb7272bdb1 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:12:59.917984 6937 tablet_copy_client.cc:523] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.921805 6937 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:12:59.940693 5210 raft_consensus.cc:1203] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:12:59.941828 6898 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.942579 6898 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:12:59.942869 6898 ts_tablet_manager.cc:1367] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.079s user 0.047s sys 0.012s
I20240501 08:12:59.943782 6898 raft_consensus.cc:348] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:59.944095 6898 raft_consensus.cc:729] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:12:59.944314 6898 consensus_queue.cc:260] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:12:59.958938 6962 tablet_copy_client.cc:523] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.962800 6962 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:12:59.980094 6898 ts_tablet_manager.cc:1398] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.037s user 0.033s sys 0.004s
I20240501 08:12:59.981581 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-4adf393d330f45ba95fb50c850f20348 received from {username='slave'} at 127.28.1.133:33215
I20240501 08:12:59.981714 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-4adf393d330f45ba95fb50c850f20348 on tablet 4adf393d330f45ba95fb50c850f20348 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:12:59.984488 6922 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.985075 6922 tablet_bootstrap.cc:492] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:59.985386 6922 ts_tablet_manager.cc:1367] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.079s user 0.044s sys 0.008s
I20240501 08:12:59.986728 6922 raft_consensus.cc:348] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.987088 6922 raft_consensus.cc:729] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:59.987313 6922 consensus_queue.cc:260] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.987884 6922 ts_tablet_manager.cc:1398] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:12:59.994065 6937 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:12:59.994315 6977 tablet_copy_client.cc:523] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:12:59.994408 6937 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:12:59.994675 6937 ts_tablet_manager.cc:1367] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.073s user 0.037s sys 0.011s
I20240501 08:12:59.995309 6937 raft_consensus.cc:348] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.995510 6937 raft_consensus.cc:729] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:12:59.995731 6937 consensus_queue.cc:260] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:12:59.996424 6937 ts_tablet_manager.cc:1398] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240501 08:12:59.996902 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-313e1bed969b4a4ba124116ba5aed741 received from {username='slave'} at 127.28.1.136:35605
I20240501 08:12:59.997056 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-313e1bed969b4a4ba124116ba5aed741 on tablet 313e1bed969b4a4ba124116ba5aed741 with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:12:59.997686 6977 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:12:59.997787 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-f75a6080c937423f8ffb9f75b96da670 received from {username='slave'} at 127.28.1.136:42911
I20240501 08:12:59.997910 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-f75a6080c937423f8ffb9f75b96da670 on tablet f75a6080c937423f8ffb9f75b96da670 with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:13:00.002375 6981 tablet_copy_client.cc:523] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:00.005934 6981 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:13:00.029083 6962 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:00.032157 6962 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:13:00.032485 6962 ts_tablet_manager.cc:1367] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.070s user 0.044s sys 0.004s
I20240501 08:13:00.033306 6962 raft_consensus.cc:348] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:00.033598 6962 raft_consensus.cc:729] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:13:00.033811 6962 consensus_queue.cc:260] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:00.034404 6962 ts_tablet_manager.cc:1398] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:13:00.035827 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-1c3672fbf92d4b54a91554f88382a5ae received from {username='slave'} at 127.28.1.133:33215
I20240501 08:13:00.035955 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-1c3672fbf92d4b54a91554f88382a5ae on tablet 1c3672fbf92d4b54a91554f88382a5ae with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:13:00.096767 6977 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:00.097291 6977 tablet_bootstrap.cc:492] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:13:00.097604 6977 ts_tablet_manager.cc:1367] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.100s user 0.051s sys 0.013s
I20240501 08:13:00.100986 6977 raft_consensus.cc:348] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:00.101284 6977 raft_consensus.cc:729] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:13:00.101490 6977 consensus_queue.cc:260] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:00.102061 6977 ts_tablet_manager.cc:1398] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240501 08:13:00.104089 6981 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:00.107097 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-36c9ad94ddc54bc6ab5021a0d297a774 received from {username='slave'} at 127.28.1.135:44357
I20240501 08:13:00.107241 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-36c9ad94ddc54bc6ab5021a0d297a774 on tablet 36c9ad94ddc54bc6ab5021a0d297a774 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:13:00.104518 6981 tablet_bootstrap.cc:492] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:13:00.110818 6981 ts_tablet_manager.cc:1367] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.105s user 0.047s sys 0.000s
I20240501 08:13:00.111598 6981 raft_consensus.cc:348] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:00.111871 6981 raft_consensus.cc:729] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:13:00.112094 6981 consensus_queue.cc:260] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:00.112828 6981 ts_tablet_manager.cc:1398] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:00.113525 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-0bd2ea6b1e714d619d7dc00d20067a59 received from {username='slave'} at 127.28.1.133:56329
I20240501 08:13:00.113672 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-0bd2ea6b1e714d619d7dc00d20067a59 on tablet 0bd2ea6b1e714d619d7dc00d20067a59 with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:13:00.117954 5078 raft_consensus.cc:1203] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.119571 6987 tablet_copy_client.cc:523] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:00.122485 6987 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:13:00.173035 6987 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:00.173470 6987 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:13:00.173744 6987 ts_tablet_manager.cc:1367] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.051s user 0.045s sys 0.000s
I20240501 08:13:00.175520 6987 raft_consensus.cc:348] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:00.175770 6987 raft_consensus.cc:729] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:13:00.175988 6987 consensus_queue.cc:260] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:00.176517 6987 ts_tablet_manager.cc:1398] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:13:00.177314 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-e62d72ab5f9b460aaa7fe7bb7ecd7b08 received from {username='slave'} at 127.28.1.135:60045
I20240501 08:13:00.177464 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-e62d72ab5f9b460aaa7fe7bb7ecd7b08 on tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:13:00.211786 5078 raft_consensus.cc:1203] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.250078 5342 raft_consensus.cc:1203] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.251482 5342 raft_consensus.cc:1203] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.302845 7032 tablet_copy_client.cc:523] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:00.305773 7032 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:13:00.336949 7032 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:00.337311 7032 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:13:00.337534 7032 ts_tablet_manager.cc:1367] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.032s user 0.028s sys 0.004s
I20240501 08:13:00.338150 7032 raft_consensus.cc:348] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:13:00.338341 7032 raft_consensus.cc:729] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:13:00.338510 7032 consensus_queue.cc:260] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:13:00.338975 7032 ts_tablet_manager.cc:1398] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:13:00.339836 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-6da0f7473ef248f2ad4f5621f3fbd25e received from {username='slave'} at 127.28.1.136:42911
I20240501 08:13:00.339995 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-6da0f7473ef248f2ad4f5621f3fbd25e on tablet 6da0f7473ef248f2ad4f5621f3fbd25e with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:13:00.345227 5474 raft_consensus.cc:1203] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.381145 5474 raft_consensus.cc:1203] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.387141 5342 raft_consensus.cc:1203] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.388198 7301 raft_consensus.cc:1050] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:13:00.388890 7301 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.391366 3149 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.391896 5474 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.392227 3279 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.392289 7299 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.392699 7300 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.392946 6451 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.395743 6451 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.396579 5474 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.399773 3279 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.399722 3149 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.401079 3003 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.412292 5391 raft_consensus.cc:1050] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:00.413177 5391 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:00.415622 3539 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.417521 5474 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.417529 3149 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.418354 3539 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.418828 6471 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.419273 5210 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.419685 7299 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.419888 6471 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.424109 6451 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.424875 7300 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.427987 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0434084d13f740368d1daae53dee9fe2 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.428903 3411 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.429114 3150 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.429543 7316 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.432644 5474 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.433012 3000 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:00.436820 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 0434084d13f740368d1daae53dee9fe2 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:00.438215 7316 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.439100 7320 tablet_replica.cc:330] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:00.439325 7320 raft_consensus.cc:2229] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.439643 7320 raft_consensus.cc:2258] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.439690 5210 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.440892 7320 ts_tablet_manager.cc:1875] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.443596 3411 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.443818 3539 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.445194 3005 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 4 to 5, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.449713 3279 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:00.451764 3410 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.452335 6471 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.455438 6471 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.458624 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bdad8a9a16414c5698f9fafb7272bdb1 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.459570 3411 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.460554 5209 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.461022 3000 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:00.462846 7316 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.464399 5210 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:00.467716 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet bdad8a9a16414c5698f9fafb7272bdb1 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:00.469707 5078 raft_consensus.cc:1203] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.479808 7323 tablet_replica.cc:330] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:00.480096 7323 raft_consensus.cc:2229] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.480530 7323 raft_consensus.cc:2258] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.481563 7320 ts_tablet_manager.cc:1888] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.481751 7320 log.cc:1196] T 0434084d13f740368d1daae53dee9fe2 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/0434084d13f740368d1daae53dee9fe2
I20240501 08:13:00.482564 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 0434084d13f740368d1daae53dee9fe2 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:13:00.483129 7323 ts_tablet_manager.cc:1875] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.490005 7323 ts_tablet_manager.cc:1888] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.490144 7323 log.cc:1196] T bdad8a9a16414c5698f9fafb7272bdb1 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:13:00.491060 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet bdad8a9a16414c5698f9fafb7272bdb1 (table rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]) successfully deleted
I20240501 08:13:00.564369 5906 raft_consensus.cc:1050] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:13:00.565016 5906 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:00.568349 5078 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.568974 3279 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.568933 7274 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.569497 3539 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.570173 7274 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.570717 7276 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.576135 5906 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.580068 5078 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.584798 3279 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.585832 3005 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.591390 3538 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.599524 3411 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:00.601496 5078 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.601675 3279 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.602154 7274 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.602492 7369 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.605598 5906 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.605981 3279 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.609531 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c3672fbf92d4b54a91554f88382a5ae with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.611790 5078 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.614012 3001 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.619374 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c3672fbf92d4b54a91554f88382a5ae with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:00.619747 7323 tablet_replica.cc:330] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:00.619940 7323 raft_consensus.cc:2229] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.620298 7323 raft_consensus.cc:2258] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.621805 7323 ts_tablet_manager.cc:1875] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.627645 7323 ts_tablet_manager.cc:1888] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.627789 7323 log.cc:1196] T 1c3672fbf92d4b54a91554f88382a5ae P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:13:00.628577 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 1c3672fbf92d4b54a91554f88382a5ae (table rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]) successfully deleted
I20240501 08:13:00.635473 7276 raft_consensus.cc:1050] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:13:00.636231 7276 consensus_queue.cc:237] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:00.638429 5342 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.638446 3150 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.638995 7274 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.639295 5906 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.639919 3538 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.640533 5906 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.669585 7274 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.670244 7388 raft_consensus.cc:1050] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:13:00.670423 3539 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.671554 7388 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:00.678628 3000 catalog_manager.cc:5484] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.678577 3150 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.681223 5341 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.687986 5341 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.688720 7020 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.688858 3279 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.689508 7020 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.689824 3411 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.690527 7020 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.698679 7391 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.699288 5341 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.703099 3279 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.704528 3001 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.708139 3410 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.715107 3150 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:00.716799 3411 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.717130 5341 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.717692 7391 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.718171 7391 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.719841 7389 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.720467 3411 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.724792 5341 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.725725 3005 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.727720 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.733136 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:00.733482 7320 tablet_replica.cc:330] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:00.733682 7320 raft_consensus.cc:2229] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.733970 7320 raft_consensus.cc:2258] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.734936 7320 ts_tablet_manager.cc:1875] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.740695 7320 ts_tablet_manager.cc:1888] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.740888 7320 log.cc:1196] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:13:00.742262 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 (table rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]) successfully deleted
I20240501 08:13:00.778388 5474 raft_consensus.cc:1203] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:00.783154 7299 raft_consensus.cc:1050] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:13:00.783816 7299 consensus_queue.cc:237] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:00.786659 5341 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.786675 3411 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.786698 3279 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.787302 7300 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.787842 7300 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.788933 7301 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.790066 7274 raft_consensus.cc:1050] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:13:00.790828 7274 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:00.801357 7301 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.802139 3150 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.802232 5078 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.802731 7276 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.803340 5906 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.804245 3280 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.804796 5906 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.804651 3410 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.814096 5341 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.815068 3005 catalog_manager.cc:5484] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.818287 3279 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.820524 3538 consensus_queue.cc:237] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:00.826762 3411 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.827526 7301 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.827912 5342 raft_consensus.cc:1261] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.828810 6451 consensus_queue.cc:1035] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.831539 7299 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.832669 3411 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.836330 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 36c9ad94ddc54bc6ab5021a0d297a774 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.838665 3005 catalog_manager.cc:5484] T 36c9ad94ddc54bc6ab5021a0d297a774 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.840003 7369 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.840586 5078 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.846971 3280 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.848369 3003 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.849372 3150 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.851720 5341 raft_consensus.cc:2932] T 36c9ad94ddc54bc6ab5021a0d297a774 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:00.861245 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 36c9ad94ddc54bc6ab5021a0d297a774 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:00.861655 7320 tablet_replica.cc:330] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:00.861842 7320 raft_consensus.cc:2229] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.862152 7320 raft_consensus.cc:2258] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.862919 7320 ts_tablet_manager.cc:1875] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.864548 3410 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75"
dest_uuid: "254a7288f105497eb60d1709036a71a9"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39052
I20240501 08:13:00.864751 3410 raft_consensus.cc:595] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:00.865192 3411 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:00.867390 3280 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.867805 5078 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.867923 7274 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.868664 7274 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.870182 7274 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.870954 7320 ts_tablet_manager.cc:1888] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.871160 7320 log.cc:1196] T 36c9ad94ddc54bc6ab5021a0d297a774 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/36c9ad94ddc54bc6ab5021a0d297a774
I20240501 08:13:00.872140 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 36c9ad94ddc54bc6ab5021a0d297a774 (table rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]) successfully deleted
I20240501 08:13:00.873809 3280 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.874013 5078 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.878185 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4adf393d330f45ba95fb50c850f20348 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.878655 3005 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:00.882491 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet 4adf393d330f45ba95fb50c850f20348 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:00.888447 7447 tablet_replica.cc:330] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:00.888669 7447 raft_consensus.cc:2229] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.888945 7447 raft_consensus.cc:2258] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.889866 7447 ts_tablet_manager.cc:1875] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.895442 7447 ts_tablet_manager.cc:1888] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.895565 7447 log.cc:1196] T 4adf393d330f45ba95fb50c850f20348 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/4adf393d330f45ba95fb50c850f20348
I20240501 08:13:00.896273 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet 4adf393d330f45ba95fb50c850f20348 (table rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]) successfully deleted
I20240501 08:13:00.901834 7389 raft_consensus.cc:1050] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:13:00.902583 7389 consensus_queue.cc:237] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.904842 3411 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.904978 3538 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.905104 5474 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.905556 7020 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.906106 7020 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.906759 7020 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.921267 7391 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.922008 3538 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.925197 3411 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.928165 3003 catalog_manager.cc:5484] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:00.930053 5474 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.935148 3149 consensus_queue.cc:237] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.937109 3411 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.937237 5474 raft_consensus.cc:1261] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.937674 7391 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.938097 7388 consensus_queue.cc:1035] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.940289 7391 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.941109 3411 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.945331 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 313e1bed969b4a4ba124116ba5aed741 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:00.946205 5473 raft_consensus.cc:2932] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.946520 3003 catalog_manager.cc:5484] T 313e1bed969b4a4ba124116ba5aed741 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 5 to 6, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.951131 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 313e1bed969b4a4ba124116ba5aed741 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:00.951475 7323 tablet_replica.cc:330] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:00.951690 7323 raft_consensus.cc:2229] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:00.951925 7323 raft_consensus.cc:2258] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:00.953279 7323 ts_tablet_manager.cc:1875] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:00.958456 7323 ts_tablet_manager.cc:1888] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:00.958701 7323 log.cc:1196] T 313e1bed969b4a4ba124116ba5aed741 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/313e1bed969b4a4ba124116ba5aed741
I20240501 08:13:00.959506 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 313e1bed969b4a4ba124116ba5aed741 (table rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]) successfully deleted
I20240501 08:13:00.963781 6471 raft_consensus.cc:1050] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:13:00.964618 6471 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.967208 3538 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.967730 5391 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.968127 5473 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.968575 6471 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.970752 3411 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:00.971392 6471 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:00.974792 7474 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.975828 3539 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.975908 3410 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.979941 5473 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.982326 3003 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.990190 3280 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:00.992100 5473 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.992925 6471 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.995039 3538 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:00.995290 5391 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.995604 7316 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:00.997246 5473 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:00.999778 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f75a6080c937423f8ffb9f75b96da670 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:01.000203 3539 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.000564 3005 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:01.005352 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet f75a6080c937423f8ffb9f75b96da670 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:01.006168 7486 tablet_replica.cc:330] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:01.006449 7486 raft_consensus.cc:2229] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:01.006889 7486 raft_consensus.cc:2258] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:01.007700 7486 ts_tablet_manager.cc:1875] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:01.014137 7486 ts_tablet_manager.cc:1888] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:01.014322 7486 log.cc:1196] T f75a6080c937423f8ffb9f75b96da670 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/f75a6080c937423f8ffb9f75b96da670
I20240501 08:13:01.015229 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet f75a6080c937423f8ffb9f75b96da670 (table rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]) successfully deleted
I20240501 08:13:01.045476 3410 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75"
dest_uuid: "254a7288f105497eb60d1709036a71a9"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39078
I20240501 08:13:01.045713 3410 raft_consensus.cc:595] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:01.062872 7369 raft_consensus.cc:979] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: : Instructing follower 1771142e14884570b63c687e982902aa to start an election
I20240501 08:13:01.063058 7369 raft_consensus.cc:1067] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Signalling peer 1771142e14884570b63c687e982902aa to start an election
I20240501 08:13:01.064033 3539 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75"
dest_uuid: "1771142e14884570b63c687e982902aa"
from {username='slave'} at 127.28.1.131:49893
I20240501 08:13:01.064323 3539 raft_consensus.cc:482] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:01.064415 3539 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:01.066547 3539 raft_consensus.cc:504] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:01.067575 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:13:01.067818 3410 raft_consensus.cc:3032] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:01.067899 3410 raft_consensus.cc:729] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:13:01.067838 3149 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:13:01.068064 3149 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:01.068171 5341 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "901e7e9545d94ee2904a29c5675ccc75" candidate_uuid: "1771142e14884570b63c687e982902aa" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3e6a44464d7748fa8c784ef795b896fb"
I20240501 08:13:01.068212 3410 consensus_queue.cc:260] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:01.068490 5341 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:01.068575 3410 raft_consensus.cc:3037] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:01.070818 3149 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 2.
I20240501 08:13:01.071316 3410 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 2.
I20240501 08:13:01.071585 3539 leader_election.cc:288] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 2 election: Requested vote from peers 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:13:01.071786 3474 leader_election.cc:302] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:13:01.072136 7300 raft_consensus.cc:2781] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:01.072280 7300 raft_consensus.cc:686] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 2 LEADER]: Becoming Leader. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:13:01.072580 7300 consensus_queue.cc:237] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:01.071982 5341 raft_consensus.cc:2454] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1771142e14884570b63c687e982902aa in term 2.
I20240501 08:13:01.075703 3000 catalog_manager.cc:5484] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa reported cstate change: term changed from 1 to 2, leader changed from 254a7288f105497eb60d1709036a71a9 (127.28.1.131) to 1771142e14884570b63c687e982902aa (127.28.1.132). New cstate: current_term: 2 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:01.078900 7299 raft_consensus.cc:1050] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:13:01.079600 7299 consensus_queue.cc:237] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:01.082131 5078 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.082645 3149 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.082705 7300 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.083154 6451 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.084427 3410 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.085059 6451 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.093305 7300 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.094009 3410 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.098469 5078 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.099200 3000 catalog_manager.cc:5484] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 4 to 5, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:01.105350 3539 consensus_queue.cc:237] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:01.107251 5078 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:01.107357 3410 raft_consensus.cc:1261] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:01.107867 7300 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.108211 7299 consensus_queue.cc:1035] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.111292 6451 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.114293 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0bd2ea6b1e714d619d7dc00d20067a59 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:01.116314 3005 catalog_manager.cc:5484] T 0bd2ea6b1e714d619d7dc00d20067a59 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:01.117770 5078 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.119342 3410 raft_consensus.cc:2932] T 0bd2ea6b1e714d619d7dc00d20067a59 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:01.120846 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet 0bd2ea6b1e714d619d7dc00d20067a59 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:01.123260 7447 tablet_replica.cc:330] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:01.123482 7447 raft_consensus.cc:2229] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:01.123659 7447 pending_rounds.cc:62] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Trying to abort 1 pending ops.
I20240501 08:13:01.123737 7447 pending_rounds.cc:68] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7022801637700190208 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "0bd2ea6b1e714d619d7dc00d20067a59" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } } }
I20240501 08:13:01.124107 7447 raft_consensus.cc:2894] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240501 08:13:01.124218 7447 raft_consensus.cc:2866] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240501 08:13:01.124357 7447 raft_consensus.cc:2258] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:01.127434 7447 ts_tablet_manager.cc:1875] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:01.141564 7447 ts_tablet_manager.cc:1888] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:01.141731 7447 log.cc:1196] T 0bd2ea6b1e714d619d7dc00d20067a59 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/0bd2ea6b1e714d619d7dc00d20067a59
I20240501 08:13:01.142462 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet 0bd2ea6b1e714d619d7dc00d20067a59 (table rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]) successfully deleted
I20240501 08:13:01.298391 5391 raft_consensus.cc:1050] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:13:01.299307 5391 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:01.301613 5473 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.301656 3539 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.301705 3150 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:01.302160 7474 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.302477 5391 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.302964 6471 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:01.309312 5391 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.310119 3150 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.310226 3539 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.314913 3000 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:01.317143 5473 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.319357 3280 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:01.321564 3539 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:01.322098 5391 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.323566 5473 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:01.324116 5391 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.324913 6471 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.325443 3538 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.328130 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6da0f7473ef248f2ad4f5621f3fbd25e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:01.328562 5473 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:01.330276 3000 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 5 to 6, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:01.333659 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet 6da0f7473ef248f2ad4f5621f3fbd25e with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:01.334012 7447 tablet_replica.cc:330] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:01.334197 7447 raft_consensus.cc:2229] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:01.334506 7447 raft_consensus.cc:2258] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:01.335520 7447 ts_tablet_manager.cc:1875] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:01.340202 7447 ts_tablet_manager.cc:1888] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:01.340320 7447 log.cc:1196] T 6da0f7473ef248f2ad4f5621f3fbd25e P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:13:01.340891 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet 6da0f7473ef248f2ad4f5621f3fbd25e (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:13:01.560353 5341 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:13:01.560959 7299 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.618842 3410 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:13:01.619421 7299 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:01.622022 3150 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:13:01.624286 7558 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:02.176911 3538 consensus_queue.cc:237] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:02.178584 5341 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240501 08:13:02.178570 3150 raft_consensus.cc:1261] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240501 08:13:02.179140 7577 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.179445 7576 consensus_queue.cc:1035] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.181926 7561 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:02.184471 3150 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:02.184471 5341 raft_consensus.cc:2932] T 901e7e9545d94ee2904a29c5675ccc75 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:02.184904 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 901e7e9545d94ee2904a29c5675ccc75 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:02.186884 3005 catalog_manager.cc:5484] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 5 to 7, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 2 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:02.197149 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet 901e7e9545d94ee2904a29c5675ccc75 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:02.197818 7583 tablet_replica.cc:330] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:02.198077 7583 raft_consensus.cc:2229] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:02.198294 7583 raft_consensus.cc:2258] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:02.199292 7583 ts_tablet_manager.cc:1875] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:02.204602 7583 ts_tablet_manager.cc:1888] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240501 08:13:02.204738 7583 log.cc:1196] T 901e7e9545d94ee2904a29c5675ccc75 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/901e7e9545d94ee2904a29c5675ccc75
I20240501 08:13:02.205463 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet 901e7e9545d94ee2904a29c5675ccc75 (table rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]) successfully deleted
I20240501 08:13:02.358282 3520 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240501 08:13:02.428398 5322 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:02.450505 6332 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:02.450755 3390 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:02.470393 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:02.481909 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:02.491696 5191 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:02.537762 3259 tablet_service.cc:1429] Tablet server has 11 leaders and 0 scanners
I20240501 08:13:02.561504 5059 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:02.636114 3280 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:02.638537 5473 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.639544 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.639705 3538 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.640601 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:02.650059 3217 consensus_peers.cc:485] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d749fd4842294f8fba36f287b7072c9d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.653298 7641 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.656463 3538 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.659299 3005 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.661346 5473 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.675022 3410 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:02.677337 5473 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.677811 3150 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.678298 7652 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.678583 7274 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:02.678869 3345 consensus_peers.cc:485] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 885de7b67f474f7bb54badfae193906d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.681339 7652 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.681998 5473 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.682651 3150 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.686875 3006 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.701799 3280 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
W20240501 08:13:02.704358 3216 consensus_peers.cc:485] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4cb5c6a514354708bff63dd5e51c7420. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.704337 5341 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.704366 3150 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.704842 7641 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.705159 7663 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.707638 7641 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.708310 5341 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.708855 3150 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.713222 3005 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.730676 3538 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:02.732831 5210 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.733168 3150 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:02.733470 3474 consensus_peers.cc:485] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a68f4cc374ad4656b1de788033ece98c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.733500 7577 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.734396 7577 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.736344 7673 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.736996 5210 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.741494 3006 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.743134 3150 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.755861 3538 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:13:02.758201 3150 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.758234 5078 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:02.758670 3475 consensus_peers.cc:485] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa -> Peer b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523): Couldn't send request to peer b1a5f0e8d89547a1ac983c6a5256eca8. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: babe0d774b24482dab935eccb99f31c1. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.758736 7673 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.759022 7577 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.761950 7577 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:13:02.762353 5079 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:13:02.764314 3150 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } } }
I20240501 08:13:02.767609 3006 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 6 to 7, NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.785154 3538 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:02.787405 5341 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.787541 3410 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.788112 7576 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.788481 7577 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.791165 7576 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.792318 3410 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
W20240501 08:13:02.795495 3477 consensus_peers.cc:485] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e21419bf69e746baa7a96fbf5eee384d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.797154 3005 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.808691 5341 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.813823 3538 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:02.816213 5473 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.816413 3149 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.816979 7576 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.817298 7577 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:02.817556 3475 consensus_peers.cc:485] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa -> Peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Couldn't send request to peer 3e6a44464d7748fa8c784ef795b896fb. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0434084d13f740368d1daae53dee9fe2. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.820072 7673 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.820617 5473 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.820822 3149 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.825994 3006 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 6 to 7, NON_VOTER 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } } }
I20240501 08:13:02.839339 3410 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:02.841784 5079 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.842362 3280 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.842383 7656 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.842904 7652 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.847147 7274 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.847690 5078 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.852077 3005 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
W20240501 08:13:02.853721 3347 consensus_peers.cc:485] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c3672fbf92d4b54a91554f88382a5ae. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.854169 3280 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.866858 3280 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:02.869247 5473 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.869310 3538 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.869872 5391 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.870246 7641 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:02.871119 3216 consensus_peers.cc:485] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f75a6080c937423f8ffb9f75b96da670. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.873360 5391 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.874181 3538 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.876288 5473 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.880610 3003 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.891943 3280 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:02.894266 5473 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:02.894412 3214 consensus_peers.cc:485] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6da0f7473ef248f2ad4f5621f3fbd25e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.894793 5391 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.894958 3538 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.895573 5391 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.897930 7641 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.898519 5473 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.899170 3539 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } } }
I20240501 08:13:02.903631 3006 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER f0af5c767c1b496cb1163870b60b066a (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.917789 3280 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:02.920049 5078 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.920120 3410 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:02.920279 3216 consensus_peers.cc:485] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 460948db7dfd4d5589d478ddd5c6c10c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.920625 7641 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.920913 7663 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.923651 5391 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.924309 5078 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.924365 3410 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.929411 3003 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:02.941882 3280 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:02.944274 5473 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
W20240501 08:13:02.944296 3216 consensus_peers.cc:485] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c927478b05c14f47b736faf52b4db9aa. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.944816 3149 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240501 08:13:02.944818 7663 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:02.945308 5391 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:02.947402 7663 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.948045 5473 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.948429 3149 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.953125 3005 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 7 to 8, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 2 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:02.967746 3149 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:02.971108 5473 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.971846 7579 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.973278 3280 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:02.973887 7751 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:02.976981 7579 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
W20240501 08:13:02.977442 3086 consensus_peers.cc:485] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a3fce8868ca74a028492122fc2762001. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:02.977617 3280 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.981280 5473 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.982582 3006 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:02.997813 3280 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:02.999922 5341 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:03.000077 3410 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:03.000257 3217 consensus_peers.cc:485] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f0c4a54ade9c4e96bb575bfbfa5d0206. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:03.000551 7663 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:03.000859 7641 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:03.003530 7641 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:03.004199 5341 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:03.004465 3410 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:03.009285 3005 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:03.022459 3280 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:03.024775 5078 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:03.025198 3149 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:03.025511 3216 consensus_peers.cc:485] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Couldn't send request to peer 8151eaf288354d60b73b3c090dd271fd. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d110b6fb2855454892ab5359ec3210b0. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:03.025511 7663 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:03.026074 7663 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:03.029163 5391 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:03.029577 3149 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:03.035044 3005 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:03.054423 5079 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } }
I20240501 08:13:03.194425 7833 ts_tablet_manager.cc:902] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:13:03.195721 7833 tablet_copy_client.cc:308] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:13:03.196449 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet 0434084d13f740368d1daae53dee9fe2 from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:44357)
I20240501 08:13:03.196637 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet 0434084d13f740368d1daae53dee9fe2 from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:44357: session id = 3e6a44464d7748fa8c784ef795b896fb-0434084d13f740368d1daae53dee9fe2
I20240501 08:13:03.199244 3560 tablet_copy_source_session.cc:205] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.200521 7833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0434084d13f740368d1daae53dee9fe2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.206120 7833 tablet_copy_client.cc:783] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.206373 7833 tablet_copy_client.cc:647] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.208137 7837 tablet_copy_client.cc:1002] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.244699 7853 ts_tablet_manager.cc:902] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.246369 7853 tablet_copy_client.cc:308] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.252768 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet d749fd4842294f8fba36f287b7072c9d from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:34297)
I20240501 08:13:03.252988 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet d749fd4842294f8fba36f287b7072c9d from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:34297: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-d749fd4842294f8fba36f287b7072c9d
I20240501 08:13:03.254899 3301 tablet_copy_source_session.cc:205] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.259893 7853 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d749fd4842294f8fba36f287b7072c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.267061 7859 ts_tablet_manager.cc:902] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:13:03.269779 7859 tablet_copy_client.cc:308] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:13:03.270586 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 885de7b67f474f7bb54badfae193906d from peer 3e6a44464d7748fa8c784ef795b896fb ({username='slave'} at 127.28.1.135:40907)
I20240501 08:13:03.270640 7853 tablet_copy_client.cc:783] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.270749 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 885de7b67f474f7bb54badfae193906d from peer 3e6a44464d7748fa8c784ef795b896fb at {username='slave'} at 127.28.1.135:40907: session id = 3e6a44464d7748fa8c784ef795b896fb-885de7b67f474f7bb54badfae193906d
I20240501 08:13:03.270965 7853 tablet_copy_client.cc:647] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.273025 7857 tablet_copy_client.cc:1002] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.273279 3430 tablet_copy_source_session.cc:205] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.274469 7859 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 885de7b67f474f7bb54badfae193906d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.278688 7859 tablet_copy_client.cc:783] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.278976 7859 tablet_copy_client.cc:647] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.279224 7864 ts_tablet_manager.cc:902] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:13:03.280498 7863 tablet_copy_client.cc:1002] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.280720 7864 tablet_copy_client.cc:308] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:13:03.285745 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet e21419bf69e746baa7a96fbf5eee384d from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:43071)
I20240501 08:13:03.285894 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet e21419bf69e746baa7a96fbf5eee384d from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:43071: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-e21419bf69e746baa7a96fbf5eee384d
I20240501 08:13:03.287878 3560 tablet_copy_source_session.cc:205] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.289196 7864 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e21419bf69e746baa7a96fbf5eee384d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.292541 7869 ts_tablet_manager.cc:902] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:13:03.293954 7869 tablet_copy_client.cc:308] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:13:03.294540 7864 tablet_copy_client.cc:783] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.294766 7864 tablet_copy_client.cc:647] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.294816 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet a68f4cc374ad4656b1de788033ece98c from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:56329)
I20240501 08:13:03.294991 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet a68f4cc374ad4656b1de788033ece98c from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:56329: session id = f0af5c767c1b496cb1163870b60b066a-a68f4cc374ad4656b1de788033ece98c
I20240501 08:13:03.296402 7868 tablet_copy_client.cc:1002] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.297307 3560 tablet_copy_source_session.cc:205] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.298365 7869 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a68f4cc374ad4656b1de788033ece98c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.302973 7869 tablet_copy_client.cc:783] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.303218 7869 tablet_copy_client.cc:647] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.304736 7873 tablet_copy_client.cc:1002] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.321173 7874 ts_tablet_manager.cc:902] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.322682 7874 tablet_copy_client.cc:308] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.323441 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 4cb5c6a514354708bff63dd5e51c7420 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:13:03.323590 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 4cb5c6a514354708bff63dd5e51c7420 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-4cb5c6a514354708bff63dd5e51c7420
I20240501 08:13:03.324344 7879 ts_tablet_manager.cc:902] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:13:03.326548 3301 tablet_copy_source_session.cc:205] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.328938 7874 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4cb5c6a514354708bff63dd5e51c7420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.331853 7879 tablet_copy_client.cc:308] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:13:03.335007 7874 tablet_copy_client.cc:783] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.335318 7874 tablet_copy_client.cc:647] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.336905 7878 tablet_copy_client.cc:1002] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.337885 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 1c3672fbf92d4b54a91554f88382a5ae from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:47885)
I20240501 08:13:03.338052 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 1c3672fbf92d4b54a91554f88382a5ae from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:47885: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:13:03.338338 7884 ts_tablet_manager.cc:902] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:13:03.339744 7884 tablet_copy_client.cc:308] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:13:03.340322 3430 tablet_copy_source_session.cc:205] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.340471 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet babe0d774b24482dab935eccb99f31c1 from peer b1a5f0e8d89547a1ac983c6a5256eca8 ({username='slave'} at 127.28.1.136:60309)
I20240501 08:13:03.340648 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet babe0d774b24482dab935eccb99f31c1 from peer b1a5f0e8d89547a1ac983c6a5256eca8 at {username='slave'} at 127.28.1.136:60309: session id = b1a5f0e8d89547a1ac983c6a5256eca8-babe0d774b24482dab935eccb99f31c1
I20240501 08:13:03.341679 7879 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c3672fbf92d4b54a91554f88382a5ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.342643 3560 tablet_copy_source_session.cc:205] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.343788 7884 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet babe0d774b24482dab935eccb99f31c1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.346699 7879 tablet_copy_client.cc:783] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.347306 7879 tablet_copy_client.cc:647] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.348731 7884 tablet_copy_client.cc:783] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.348975 7884 tablet_copy_client.cc:647] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.349027 7883 tablet_copy_client.cc:1002] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.350876 7888 tablet_copy_client.cc:1002] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.387243 7889 ts_tablet_manager.cc:902] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.388690 7889 tablet_copy_client.cc:308] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.389321 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet c927478b05c14f47b736faf52b4db9aa from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:13:03.389485 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet c927478b05c14f47b736faf52b4db9aa from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-c927478b05c14f47b736faf52b4db9aa
I20240501 08:13:03.391860 3301 tablet_copy_source_session.cc:205] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.393149 7889 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c927478b05c14f47b736faf52b4db9aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.395159 7894 ts_tablet_manager.cc:902] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.396823 7894 tablet_copy_client.cc:308] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.397505 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet f75a6080c937423f8ffb9f75b96da670 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:13:03.397645 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet f75a6080c937423f8ffb9f75b96da670 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-f75a6080c937423f8ffb9f75b96da670
I20240501 08:13:03.398375 7889 tablet_copy_client.cc:783] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.398653 7889 tablet_copy_client.cc:647] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.400050 3301 tablet_copy_source_session.cc:205] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.400211 7893 tablet_copy_client.cc:1002] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.401144 7894 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75a6080c937423f8ffb9f75b96da670. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.405196 7894 tablet_copy_client.cc:783] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.405414 7894 tablet_copy_client.cc:647] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.406807 7898 tablet_copy_client.cc:1002] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.408358 7899 ts_tablet_manager.cc:902] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:13:03.409751 7899 tablet_copy_client.cc:308] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:13:03.412248 7904 ts_tablet_manager.cc:902] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.413789 7904 tablet_copy_client.cc:308] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.414489 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 6da0f7473ef248f2ad4f5621f3fbd25e from peer f0af5c767c1b496cb1163870b60b066a ({username='slave'} at 127.28.1.133:47763)
I20240501 08:13:03.414637 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 6da0f7473ef248f2ad4f5621f3fbd25e from peer f0af5c767c1b496cb1163870b60b066a at {username='slave'} at 127.28.1.133:47763: session id = f0af5c767c1b496cb1163870b60b066a-6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:13:03.416432 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet a3fce8868ca74a028492122fc2762001 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:49567)
I20240501 08:13:03.416605 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet a3fce8868ca74a028492122fc2762001 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:49567: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-a3fce8868ca74a028492122fc2762001
I20240501 08:13:03.417367 3301 tablet_copy_source_session.cc:205] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.418752 7904 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6da0f7473ef248f2ad4f5621f3fbd25e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.418958 3170 tablet_copy_source_session.cc:205] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.420207 7899 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a3fce8868ca74a028492122fc2762001. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.423300 7904 tablet_copy_client.cc:783] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.423621 7904 tablet_copy_client.cc:647] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.424676 7899 tablet_copy_client.cc:783] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.424899 7899 tablet_copy_client.cc:647] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.425644 7908 tablet_copy_client.cc:1002] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.426522 7903 tablet_copy_client.cc:1002] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.443354 7914 ts_tablet_manager.cc:902] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.445638 7914 tablet_copy_client.cc:308] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.446343 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 460948db7dfd4d5589d478ddd5c6c10c from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:13:03.446523 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 460948db7dfd4d5589d478ddd5c6c10c from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:13:03.449194 3301 tablet_copy_source_session.cc:205] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.450445 7914 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 460948db7dfd4d5589d478ddd5c6c10c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.455047 7914 tablet_copy_client.cc:783] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.455276 7914 tablet_copy_client.cc:647] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.457064 7918 tablet_copy_client.cc:1002] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.573377 7989 ts_tablet_manager.cc:902] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.588560 7992 ts_tablet_manager.cc:902] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:03.617280 7989 tablet_copy_client.cc:308] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.618233 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet d110b6fb2855454892ab5359ec3210b0 from peer 8151eaf288354d60b73b3c090dd271fd ({username='slave'} at 127.28.1.134:50227)
I20240501 08:13:03.618429 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet d110b6fb2855454892ab5359ec3210b0 from peer 8151eaf288354d60b73b3c090dd271fd at {username='slave'} at 127.28.1.134:50227: session id = 8151eaf288354d60b73b3c090dd271fd-d110b6fb2855454892ab5359ec3210b0
I20240501 08:13:03.621354 3301 tablet_copy_source_session.cc:205] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.622906 7989 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d110b6fb2855454892ab5359ec3210b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.623409 7992 tablet_copy_client.cc:308] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:03.624150 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:34297)
I20240501 08:13:03.624294 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet f0c4a54ade9c4e96bb575bfbfa5d0206 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:34297: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:13:03.626818 3301 tablet_copy_source_session.cc:205] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:03.627954 7989 tablet_copy_client.cc:783] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.628016 7992 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f0c4a54ade9c4e96bb575bfbfa5d0206. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:03.628284 7989 tablet_copy_client.cc:647] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.629834 7996 tablet_copy_client.cc:1002] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:03.632526 7992 tablet_copy_client.cc:783] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:03.632735 7992 tablet_copy_client.cc:647] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:03.634301 7998 tablet_copy_client.cc:1002] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:04.708500 7833 tablet_copy_client.cc:523] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.711171 7833 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:13:04.749145 7833 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.749596 7833 tablet_bootstrap.cc:492] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:13:04.749917 7833 ts_tablet_manager.cc:1367] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.039s user 0.039s sys 0.000s
I20240501 08:13:04.750694 7833 raft_consensus.cc:348] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:04.750943 7833 raft_consensus.cc:729] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:13:04.751137 7833 consensus_queue.cc:260] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:04.751871 7833 ts_tablet_manager.cc:1398] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:13:04.752538 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-0434084d13f740368d1daae53dee9fe2 received from {username='slave'} at 127.28.1.135:44357
I20240501 08:13:04.752699 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-0434084d13f740368d1daae53dee9fe2 on tablet 0434084d13f740368d1daae53dee9fe2 with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:13:04.770402 5341 raft_consensus.cc:1203] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:04.773514 7853 tablet_copy_client.cc:523] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.777089 7853 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:04.780839 7859 tablet_copy_client.cc:523] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.783360 7859 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap starting.
I20240501 08:13:04.793761 7853 log.cc:824] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:04.796797 7864 tablet_copy_client.cc:523] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.800503 7864 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:04.805104 7869 tablet_copy_client.cc:523] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.809042 7869 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:13:04.830054 7859 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.830514 7859 tablet_bootstrap.cc:492] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Bootstrap complete.
I20240501 08:13:04.830797 7859 ts_tablet_manager.cc:1367] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Time spent bootstrapping tablet: real 0.047s user 0.038s sys 0.007s
I20240501 08:13:04.831596 7859 raft_consensus.cc:348] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:04.831887 7859 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Initialized, Role: LEARNER
I20240501 08:13:04.832120 7859 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: NON_VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: true } }
I20240501 08:13:04.832736 7859 ts_tablet_manager.cc:1398] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:13:04.834233 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session 3e6a44464d7748fa8c784ef795b896fb-885de7b67f474f7bb54badfae193906d received from {username='slave'} at 127.28.1.135:40907
I20240501 08:13:04.834362 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session 3e6a44464d7748fa8c784ef795b896fb-885de7b67f474f7bb54badfae193906d on tablet 885de7b67f474f7bb54badfae193906d with peer 3e6a44464d7748fa8c784ef795b896fb
I20240501 08:13:04.838410 7874 tablet_copy_client.cc:523] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.842018 7874 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:13:04.849421 7879 tablet_copy_client.cc:523] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.851253 7884 tablet_copy_client.cc:523] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.853358 7879 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:04.854919 7884 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap starting.
I20240501 08:13:04.865060 7869 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.865489 7869 tablet_bootstrap.cc:492] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:13:04.865805 7869 ts_tablet_manager.cc:1367] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.057s user 0.041s sys 0.010s
I20240501 08:13:04.865895 7864 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.866516 7864 tablet_bootstrap.cc:492] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:04.866573 7869 raft_consensus.cc:348] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:04.866868 7869 raft_consensus.cc:729] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:13:04.867071 7869 consensus_queue.cc:260] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:04.867108 7864 ts_tablet_manager.cc:1367] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.067s user 0.052s sys 0.009s
I20240501 08:13:04.873901 7864 raft_consensus.cc:348] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.874421 7864 raft_consensus.cc:729] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:04.874982 7869 ts_tablet_manager.cc:1398] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.009s user 0.009s sys 0.000s
I20240501 08:13:04.875028 7864 consensus_queue.cc:260] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.877823 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-a68f4cc374ad4656b1de788033ece98c received from {username='slave'} at 127.28.1.133:56329
I20240501 08:13:04.877990 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-a68f4cc374ad4656b1de788033ece98c on tablet a68f4cc374ad4656b1de788033ece98c with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:13:04.885406 7853 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.893793 7853 tablet_bootstrap.cc:492] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:04.894214 7853 ts_tablet_manager.cc:1367] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.117s user 0.057s sys 0.005s
I20240501 08:13:04.900569 7889 tablet_copy_client.cc:523] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.904167 7889 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:13:04.904315 7853 raft_consensus.cc:348] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.904670 7853 raft_consensus.cc:729] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:04.904945 7853 consensus_queue.cc:260] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.907851 7894 tablet_copy_client.cc:523] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.909422 7864 ts_tablet_manager.cc:1398] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.042s user 0.023s sys 0.016s
I20240501 08:13:04.910961 5341 raft_consensus.cc:1203] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:04.911633 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-e21419bf69e746baa7a96fbf5eee384d received from {username='slave'} at 127.28.1.137:43071
I20240501 08:13:04.911784 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-e21419bf69e746baa7a96fbf5eee384d on tablet e21419bf69e746baa7a96fbf5eee384d with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:04.915549 7894 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:13:04.922456 7879 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.922878 7879 tablet_bootstrap.cc:492] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:04.923147 7879 ts_tablet_manager.cc:1367] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.070s user 0.034s sys 0.010s
I20240501 08:13:04.924434 7879 raft_consensus.cc:348] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.924700 7879 raft_consensus.cc:729] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:04.924904 7879 consensus_queue.cc:260] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:04.927405 7904 tablet_copy_client.cc:523] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.927853 7879 ts_tablet_manager.cc:1398] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:04.929531 7853 ts_tablet_manager.cc:1398] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.035s user 0.001s sys 0.007s
I20240501 08:13:04.930919 7904 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Bootstrap starting.
I20240501 08:13:04.932492 7899 tablet_copy_client.cc:523] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.932685 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-1c3672fbf92d4b54a91554f88382a5ae received from {username='slave'} at 127.28.1.137:47885
I20240501 08:13:04.932803 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-1c3672fbf92d4b54a91554f88382a5ae on tablet 1c3672fbf92d4b54a91554f88382a5ae with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:04.935293 7899 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:04.936756 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-d749fd4842294f8fba36f287b7072c9d received from {username='slave'} at 127.28.1.137:34297
I20240501 08:13:04.936901 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-d749fd4842294f8fba36f287b7072c9d on tablet d749fd4842294f8fba36f287b7072c9d with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:04.954447 7874 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:04.955025 7874 tablet_bootstrap.cc:492] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:13:04.955396 7874 ts_tablet_manager.cc:1367] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.113s user 0.045s sys 0.009s
I20240501 08:13:04.956348 7874 raft_consensus.cc:348] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:04.956765 7874 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:13:04.957036 7874 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:04.957840 7914 tablet_copy_client.cc:523] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:04.962311 7914 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:13:04.991497 6352 raft_consensus.cc:1203] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:04.996631 7874 ts_tablet_manager.cc:1398] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.041s user 0.023s sys 0.016s
I20240501 08:13:05.005697 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-4cb5c6a514354708bff63dd5e51c7420 received from {username='slave'} at 127.28.1.134:50227
I20240501 08:13:05.005882 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-4cb5c6a514354708bff63dd5e51c7420 on tablet 4cb5c6a514354708bff63dd5e51c7420 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:13:05.006214 7884 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.008144 7884 tablet_bootstrap.cc:492] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Bootstrap complete.
I20240501 08:13:05.008522 7884 ts_tablet_manager.cc:1367] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent bootstrapping tablet: real 0.154s user 0.050s sys 0.007s
I20240501 08:13:05.009382 7884 raft_consensus.cc:348] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:13:05.009696 7884 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b1a5f0e8d89547a1ac983c6a5256eca8, State: Initialized, Role: LEARNER
I20240501 08:13:05.010936 7884 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: true } }
I20240501 08:13:05.037214 7899 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.037707 7899 tablet_bootstrap.cc:492] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:05.038000 7899 ts_tablet_manager.cc:1367] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.103s user 0.046s sys 0.008s
I20240501 08:13:05.058629 7884 ts_tablet_manager.cc:1398] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8: Time spent starting tablet: real 0.050s user 0.029s sys 0.018s
I20240501 08:13:05.059676 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-babe0d774b24482dab935eccb99f31c1 received from {username='slave'} at 127.28.1.136:60309
I20240501 08:13:05.059859 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session b1a5f0e8d89547a1ac983c6a5256eca8-babe0d774b24482dab935eccb99f31c1 on tablet babe0d774b24482dab935eccb99f31c1 with peer b1a5f0e8d89547a1ac983c6a5256eca8
I20240501 08:13:05.063460 7899 raft_consensus.cc:348] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:05.063819 7899 raft_consensus.cc:729] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:05.064047 7899 consensus_queue.cc:260] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:05.068996 7899 ts_tablet_manager.cc:1398] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.031s user 0.004s sys 0.000s
I20240501 08:13:05.070070 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-a3fce8868ca74a028492122fc2762001 received from {username='slave'} at 127.28.1.137:49567
I20240501 08:13:05.070222 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-a3fce8868ca74a028492122fc2762001 on tablet a3fce8868ca74a028492122fc2762001 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:05.076460 7904 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.076956 7904 tablet_bootstrap.cc:492] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Bootstrap complete.
I20240501 08:13:05.077241 7904 ts_tablet_manager.cc:1367] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Time spent bootstrapping tablet: real 0.146s user 0.049s sys 0.004s
I20240501 08:13:05.078135 7904 raft_consensus.cc:348] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:05.078413 7904 raft_consensus.cc:729] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Initialized, Role: LEARNER
I20240501 08:13:05.080507 7904 consensus_queue.cc:260] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: true } }
I20240501 08:13:05.084283 7904 ts_tablet_manager.cc:1398] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
I20240501 08:13:05.084333 7914 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.084761 7914 tablet_bootstrap.cc:492] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:13:05.085033 7914 ts_tablet_manager.cc:1367] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.123s user 0.049s sys 0.004s
I20240501 08:13:05.085857 7914 raft_consensus.cc:348] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.086151 7914 raft_consensus.cc:729] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:13:05.086354 7914 consensus_queue.cc:260] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.086717 7889 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.087165 7889 tablet_bootstrap.cc:492] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:13:05.087492 7889 ts_tablet_manager.cc:1367] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.183s user 0.060s sys 0.006s
I20240501 08:13:05.088042 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session f0af5c767c1b496cb1163870b60b066a-6da0f7473ef248f2ad4f5621f3fbd25e received from {username='slave'} at 127.28.1.133:47763
I20240501 08:13:05.088189 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session f0af5c767c1b496cb1163870b60b066a-6da0f7473ef248f2ad4f5621f3fbd25e on tablet 6da0f7473ef248f2ad4f5621f3fbd25e with peer f0af5c767c1b496cb1163870b60b066a
I20240501 08:13:05.088335 7914 ts_tablet_manager.cc:1398] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:05.090476 7889 raft_consensus.cc:348] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.090727 7889 raft_consensus.cc:729] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:13:05.090945 7889 consensus_queue.cc:260] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.091511 7889 ts_tablet_manager.cc:1398] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.003s user 0.000s sys 0.000s
I20240501 08:13:05.092306 3300 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-c927478b05c14f47b736faf52b4db9aa received from {username='slave'} at 127.28.1.134:50227
I20240501 08:13:05.092438 3300 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-c927478b05c14f47b736faf52b4db9aa on tablet c927478b05c14f47b736faf52b4db9aa with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:13:05.093362 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-460948db7dfd4d5589d478ddd5c6c10c received from {username='slave'} at 127.28.1.134:50227
I20240501 08:13:05.093488 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-460948db7dfd4d5589d478ddd5c6c10c on tablet 460948db7dfd4d5589d478ddd5c6c10c with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:13:05.096500 7894 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.097013 7894 tablet_bootstrap.cc:492] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:13:05.097362 7894 ts_tablet_manager.cc:1367] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.182s user 0.057s sys 0.004s
I20240501 08:13:05.098245 7894 raft_consensus.cc:348] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.098929 7894 raft_consensus.cc:729] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:13:05.099134 7894 consensus_queue.cc:260] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.099612 7894 ts_tablet_manager.cc:1398] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:13:05.100741 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-f75a6080c937423f8ffb9f75b96da670 received from {username='slave'} at 127.28.1.134:50227
I20240501 08:13:05.100883 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-f75a6080c937423f8ffb9f75b96da670 on tablet f75a6080c937423f8ffb9f75b96da670 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:13:05.118077 5210 raft_consensus.cc:1203] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.130244 7989 tablet_copy_client.cc:523] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:05.133294 7989 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap starting.
I20240501 08:13:05.134737 7992 tablet_copy_client.cc:523] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:05.138250 7992 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:05.168435 5473 raft_consensus.cc:1203] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.184509 7989 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.185150 7989 tablet_bootstrap.cc:492] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Bootstrap complete.
I20240501 08:13:05.185515 7989 ts_tablet_manager.cc:1367] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Time spent bootstrapping tablet: real 0.052s user 0.035s sys 0.014s
I20240501 08:13:05.186360 7989 raft_consensus.cc:348] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.186658 7989 raft_consensus.cc:729] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Initialized, Role: LEARNER
I20240501 08:13:05.186879 7989 consensus_queue.cc:260] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } }
I20240501 08:13:05.187705 7989 ts_tablet_manager.cc:1398] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:13:05.189148 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 8151eaf288354d60b73b3c090dd271fd-d110b6fb2855454892ab5359ec3210b0 received from {username='slave'} at 127.28.1.134:50227
I20240501 08:13:05.189289 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 8151eaf288354d60b73b3c090dd271fd-d110b6fb2855454892ab5359ec3210b0 on tablet d110b6fb2855454892ab5359ec3210b0 with peer 8151eaf288354d60b73b3c090dd271fd
I20240501 08:13:05.201030 7992 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:05.201521 7992 tablet_bootstrap.cc:492] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:05.201812 7992 ts_tablet_manager.cc:1367] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.064s user 0.038s sys 0.012s
I20240501 08:13:05.202566 7992 raft_consensus.cc:348] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:05.202845 7992 raft_consensus.cc:729] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:05.203042 7992 consensus_queue.cc:260] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:05.203523 7992 ts_tablet_manager.cc:1398] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:13:05.204187 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-f0c4a54ade9c4e96bb575bfbfa5d0206 received from {username='slave'} at 127.28.1.137:34297
I20240501 08:13:05.204314 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-f0c4a54ade9c4e96bb575bfbfa5d0206 on tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:05.212898 6352 raft_consensus.cc:1203] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.320201 8335 raft_consensus.cc:1050] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:13:05.321038 8335 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:05.323999 5473 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.324271 5341 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.324424 3150 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.324903 7576 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.325206 8288 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.325839 8335 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.330549 8335 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.331236 3150 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.334887 5341 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.336472 3005 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.339570 5473 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.344642 6352 raft_consensus.cc:1203] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.351924 3539 consensus_queue.cc:237] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:05.353632 5473 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.353747 5341 raft_consensus.cc:1261] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.354205 8288 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.354558 8335 consensus_queue.cc:1035] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.356706 8336 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.357496 5473 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.357467 5341 raft_consensus.cc:2932] T 0434084d13f740368d1daae53dee9fe2 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.360785 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0434084d13f740368d1daae53dee9fe2 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.362216 3003 catalog_manager.cc:5484] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.365824 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet 0434084d13f740368d1daae53dee9fe2 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:05.366470 8345 tablet_replica.cc:330] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:05.366650 8345 raft_consensus.cc:2229] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:05.366852 8345 raft_consensus.cc:2258] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:05.367662 8345 ts_tablet_manager.cc:1875] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:05.372377 8345 ts_tablet_manager.cc:1888] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:05.372555 8345 log.cc:1196] T 0434084d13f740368d1daae53dee9fe2 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/0434084d13f740368d1daae53dee9fe2
I20240501 08:13:05.373432 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet 0434084d13f740368d1daae53dee9fe2 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:13:05.389026 5210 raft_consensus.cc:1203] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.397132 5210 raft_consensus.cc:1203] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.433120 5079 raft_consensus.cc:1203] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.436468 6352 raft_consensus.cc:1203] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.467190 5210 raft_consensus.cc:1203] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.483420 5210 raft_consensus.cc:1203] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240501 08:13:05.503746 8303 raft_consensus.cc:1050] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER 3e6a44464d7748fa8c784ef795b896fb to VOTER
I20240501 08:13:05.504608 8303 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:05.507665 5473 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.508338 5341 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.509320 3150 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.510339 8356 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.510979 8155 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.511574 8155 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.520419 8363 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.521840 5473 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.527321 3003 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.530292 3149 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.531430 5341 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:05.564445 7772 raft_consensus.cc:1050] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:05.565178 7772 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.567795 5473 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.567895 3539 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.568776 6352 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.569061 7772 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.570978 8389 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.571501 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.574858 5079 raft_consensus.cc:1203] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.597645 7772 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.598855 3539 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.601532 5473 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.603017 3001 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:05.610040 8389 raft_consensus.cc:1050] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:05.610721 8389 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.613373 5210 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.613912 5079 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.614120 7772 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.614670 5391 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.617295 3410 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.618480 5391 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.621111 6351 raft_consensus.cc:1203] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:05.629673 5391 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.633572 6352 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.638548 3280 consensus_queue.cc:237] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.639605 5079 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.640153 5209 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.643088 6351 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.644464 5473 raft_consensus.cc:1261] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.644632 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.645303 5391 consensus_queue.cc:1035] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.647779 3006 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd reported cstate change: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.650643 8389 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.653996 3280 consensus_queue.cc:237] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.655243 6352 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.655567 5473 raft_consensus.cc:2932] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.655958 5210 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.656514 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d749fd4842294f8fba36f287b7072c9d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.657349 7772 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.658563 3006 catalog_manager.cc:5484] T d749fd4842294f8fba36f287b7072c9d P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.659742 7772 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.660842 5079 raft_consensus.cc:1261] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.663348 5209 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.663569 8387 consensus_queue.cc:1035] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.663779 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet d749fd4842294f8fba36f287b7072c9d with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:05.666306 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 460948db7dfd4d5589d478ddd5c6c10c with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.667162 8402 tablet_replica.cc:330] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:05.667456 8402 raft_consensus.cc:2229] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:05.667874 8402 raft_consensus.cc:2258] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:05.668876 3001 catalog_manager.cc:5484] T 460948db7dfd4d5589d478ddd5c6c10c P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.669845 8402 ts_tablet_manager.cc:1875] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:05.671145 5079 raft_consensus.cc:2932] T 460948db7dfd4d5589d478ddd5c6c10c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.675325 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet 460948db7dfd4d5589d478ddd5c6c10c with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:05.677198 8402 ts_tablet_manager.cc:1888] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:05.677412 8402 log.cc:1196] T d749fd4842294f8fba36f287b7072c9d P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/d749fd4842294f8fba36f287b7072c9d
I20240501 08:13:05.678303 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet d749fd4842294f8fba36f287b7072c9d (table rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]) successfully deleted
I20240501 08:13:05.685663 7576 raft_consensus.cc:1050] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER b1a5f0e8d89547a1ac983c6a5256eca8 to VOTER
I20240501 08:13:05.686575 7576 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:05.689294 3149 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.689554 5473 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.690052 7576 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.690506 5079 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.691016 8288 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.692075 8403 tablet_replica.cc:330] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:05.692332 8403 raft_consensus.cc:2229] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:05.692641 8403 pending_rounds.cc:62] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Trying to abort 1 pending ops.
I20240501 08:13:05.692701 8336 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.692773 8403 pending_rounds.cc:68] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Aborting op as it isn't in flight: id { term: 1 index: 8 } timestamp: 7022801656259788800 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "460948db7dfd4d5589d478ddd5c6c10c" old_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: NON_VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: true } } } new_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } } }
I20240501 08:13:05.693321 8403 raft_consensus.cc:2894] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Aborting config change with OpId 1.8: Aborted: Op aborted
I20240501 08:13:05.693500 8403 raft_consensus.cc:2866] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240501 08:13:05.693672 8403 raft_consensus.cc:2258] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:05.694614 8403 ts_tablet_manager.cc:1875] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:05.702036 8336 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:05.704025 8403 ts_tablet_manager.cc:1888] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:05.704226 8403 log.cc:1196] T 460948db7dfd4d5589d478ddd5c6c10c P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:13:05.705092 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet 460948db7dfd4d5589d478ddd5c6c10c (table rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]) successfully deleted
I20240501 08:13:05.705729 5474 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:05.706856 5078 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:05.707773 3001 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 7 to 8, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.708436 3149 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:05.735733 3539 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1"
dest_uuid: "1771142e14884570b63c687e982902aa"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:58914
I20240501 08:13:05.736023 3539 raft_consensus.cc:595] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:05.779628 7772 raft_consensus.cc:1050] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:05.780508 7772 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.782963 5341 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.783087 3410 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.783565 8389 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.783892 6352 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.784040 8387 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.784471 8389 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.788136 8387 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.789134 6352 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.793677 3001 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.794227 3410 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.795091 5341 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.801717 3280 consensus_queue.cc:237] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.804298 6352 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.804853 8387 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.805853 5341 raft_consensus.cc:1261] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.806855 8387 consensus_queue.cc:1035] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.809212 7772 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.809963 5341 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.809912 6352 raft_consensus.cc:2932] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.812780 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.815419 3001 catalog_manager.cc:5484] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.819295 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet f0c4a54ade9c4e96bb575bfbfa5d0206 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:05.819662 8403 tablet_replica.cc:330] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:05.819871 8403 raft_consensus.cc:2229] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:05.820240 8403 raft_consensus.cc:2258] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:05.821099 8403 ts_tablet_manager.cc:1875] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:05.825412 3410 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "885de7b67f474f7bb54badfae193906d"
dest_uuid: "254a7288f105497eb60d1709036a71a9"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:47222
I20240501 08:13:05.825628 3410 raft_consensus.cc:595] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:05.826035 8403 ts_tablet_manager.cc:1888] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:05.826236 8403 log.cc:1196] T f0c4a54ade9c4e96bb575bfbfa5d0206 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:13:05.827032 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet f0c4a54ade9c4e96bb575bfbfa5d0206 (table rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]) successfully deleted
I20240501 08:13:05.847548 3410 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:05.849901 3280 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:05.850031 5078 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:05.850059 3347 consensus_peers.cc:485] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4adf393d330f45ba95fb50c850f20348. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:05.850446 8303 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:05.850737 8363 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:05.853303 8356 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:05.853899 3280 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:05.854001 5078 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:05.858433 3006 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:05.867657 8288 raft_consensus.cc:1050] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:05.868599 8288 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.872956 6352 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.872992 3410 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.873149 5341 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.873739 8335 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.874421 7576 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.874892 7576 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.877024 8389 raft_consensus.cc:1050] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:05.877807 8389 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.880089 8335 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.880704 3410 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.886557 3003 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.886646 5341 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.890087 5209 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:05.891031 5391 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.893040 5391 raft_consensus.cc:1050] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:05.893851 5391 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.897576 5341 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.898195 8387 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.899322 3149 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.899886 5210 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.900434 8387 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.903232 8387 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.904722 5474 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:05.906126 3150 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:05.907866 8480 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.908424 7772 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.911613 6352 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.914270 8387 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.914737 3539 consensus_queue.cc:237] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.917086 8288 raft_consensus.cc:1050] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:13:05.917330 3150 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.917923 8288 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:05.920470 5342 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.921106 7576 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.921640 6352 raft_consensus.cc:1261] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:05.922966 7576 consensus_queue.cc:1035] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:05.924392 7576 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.923430 3001 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 8 to 9, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.928602 5342 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.928519 5210 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.933007 5079 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.933781 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e21419bf69e746baa7a96fbf5eee384d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.934095 6351 raft_consensus.cc:2932] T e21419bf69e746baa7a96fbf5eee384d P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:05.934825 3006 catalog_manager.cc:5484] T e21419bf69e746baa7a96fbf5eee384d P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:05.938650 8335 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.938939 5209 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.938987 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet e21419bf69e746baa7a96fbf5eee384d with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:05.939419 8403 tablet_replica.cc:330] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:05.939592 3150 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.939617 8403 raft_consensus.cc:2229] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:05.940080 8403 raft_consensus.cc:2258] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:05.941067 8403 ts_tablet_manager.cc:1875] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:05.941524 8335 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.942081 8335 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.947582 5474 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.954066 7576 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:05.954957 3280 consensus_queue.cc:237] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.955224 5391 raft_consensus.cc:1050] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:05.955899 5391 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:05.958489 5078 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:05.959635 5209 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:05.960212 8387 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:05.964942 8403 ts_tablet_manager.cc:1888] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:05.965330 8403 log.cc:1196] T e21419bf69e746baa7a96fbf5eee384d P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/e21419bf69e746baa7a96fbf5eee384d
I20240501 08:13:05.966176 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet e21419bf69e746baa7a96fbf5eee384d (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:13:05.967268 5209 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.967131 8387 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.968190 5474 raft_consensus.cc:1261] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:05.968429 5209 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P 8151eaf288354d60b73b3c090dd271fd [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.969128 5473 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.970301 7772 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.972218 7772 consensus_queue.cc:1035] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.002s
I20240501 08:13:05.973731 3150 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:05.974082 5472 raft_consensus.cc:2932] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.975548 3003 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.978377 8387 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.980480 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c927478b05c14f47b736faf52b4db9aa with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:05.981731 3539 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:05.981541 3000 catalog_manager.cc:5484] T c927478b05c14f47b736faf52b4db9aa P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 9 to 10, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 2 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:05.982311 7772 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:05.985314 8303 raft_consensus.cc:1050] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:05.986171 8303 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:05.989584 5210 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:05.991133 8363 raft_consensus.cc:979] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: : Instructing follower 3e6a44464d7748fa8c784ef795b896fb to start an election
I20240501 08:13:05.991261 8363 raft_consensus.cc:1067] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Signalling peer 3e6a44464d7748fa8c784ef795b896fb to start an election
I20240501 08:13:05.992141 5340 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "885de7b67f474f7bb54badfae193906d"
dest_uuid: "3e6a44464d7748fa8c784ef795b896fb"
from {username='slave'} at 127.28.1.131:49147
I20240501 08:13:05.992458 5340 raft_consensus.cc:482] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:05.992594 5340 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:05.995123 8387 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:05.995456 5340 raft_consensus.cc:504] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:05.998154 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "3e6a44464d7748fa8c784ef795b896fb" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:13:05.998420 3150 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:05.998361 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "3e6a44464d7748fa8c784ef795b896fb" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:13:05.998560 3410 raft_consensus.cc:3032] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:05.998651 3410 raft_consensus.cc:729] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 254a7288f105497eb60d1709036a71a9, State: Running, Role: LEADER
I20240501 08:13:05.998960 3410 consensus_queue.cc:260] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:05.999390 3410 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.001250 3150 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e6a44464d7748fa8c784ef795b896fb in term 2.
I20240501 08:13:06.002256 3410 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e6a44464d7748fa8c784ef795b896fb in term 2.
I20240501 08:13:06.003522 5275 leader_election.cc:302] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 3e6a44464d7748fa8c784ef795b896fb, 466a9d70da1b4b1e8579e40447ce0c25; no voters:
I20240501 08:13:06.003484 3538 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.003948 8235 raft_consensus.cc:2781] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:06.004238 5472 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.008426 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet c927478b05c14f47b736faf52b4db9aa with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:06.010524 8502 tablet_replica.cc:330] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:06.011189 5391 raft_consensus.cc:1050] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 8151eaf288354d60b73b3c090dd271fd to VOTER
I20240501 08:13:06.012118 5391 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.014319 3004 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.021495 8502 raft_consensus.cc:2229] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.022022 8502 raft_consensus.cc:2258] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.024264 5210 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.024811 5078 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.025319 3150 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.025606 8480 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.028386 8389 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.028904 8389 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.035820 5340 leader_election.cc:288] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [CANDIDATE]: Term 2 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523)
I20240501 08:13:06.040397 8502 ts_tablet_manager.cc:1875] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.048858 3538 consensus_queue.cc:237] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:06.050987 5210 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.051065 5079 raft_consensus.cc:1261] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.051586 8336 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.052239 7576 consensus_queue.cc:1035] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.055167 7576 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.059587 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a68f4cc374ad4656b1de788033ece98c with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.060011 5079 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.060231 8480 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.066488 5210 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.066918 3149 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.071527 5210 raft_consensus.cc:2932] T a68f4cc374ad4656b1de788033ece98c P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.073861 3001 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.073984 7772 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.077811 3005 catalog_manager.cc:5484] T a68f4cc374ad4656b1de788033ece98c P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.079682 5210 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.080747 3150 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.082943 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet a68f4cc374ad4656b1de788033ece98c with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:06.087889 8502 ts_tablet_manager.cc:1888] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:06.088143 8502 log.cc:1196] T c927478b05c14f47b736faf52b4db9aa P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/c927478b05c14f47b736faf52b4db9aa
I20240501 08:13:06.087880 3007 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.088194 6351 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.088810 8155 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.095153 5209 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.096419 8235 raft_consensus.cc:686] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 2 LEADER]: Becoming Leader. State: Replica: 3e6a44464d7748fa8c784ef795b896fb, State: Running, Role: LEADER
I20240501 08:13:06.098829 5079 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.099601 3280 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.099539 8155 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.099547 8235 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:06.100176 8155 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.103071 3279 consensus_queue.cc:237] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.105455 5472 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.106180 8480 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.106570 5211 raft_consensus.cc:1261] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.107290 5391 consensus_queue.cc:1035] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.108520 5077 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.112700 8389 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.114277 5209 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.117636 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f75a6080c937423f8ffb9f75b96da670 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.119211 5340 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 8151eaf288354d60b73b3c090dd271fd (127.28.1.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.119824 5472 raft_consensus.cc:2932] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.121722 3001 catalog_manager.cc:5484] T f75a6080c937423f8ffb9f75b96da670 P 8151eaf288354d60b73b3c090dd271fd reported cstate change: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.124495 8203 raft_consensus.cc:1050] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:06.128794 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet f75a6080c937423f8ffb9f75b96da670 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:06.129155 8402 tablet_replica.cc:330] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:06.129354 8402 raft_consensus.cc:2229] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.129186 8203 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:06.129608 8402 raft_consensus.cc:2258] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.129916 8502 tablet_replica.cc:330] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:06.129936 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet c927478b05c14f47b736faf52b4db9aa (table rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]) successfully deleted
I20240501 08:13:06.130096 8502 raft_consensus.cc:2229] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.130724 8402 ts_tablet_manager.cc:1875] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.135268 3003 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: term changed from 1 to 2, leader changed from 254a7288f105497eb60d1709036a71a9 (127.28.1.131) to 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135). New cstate: current_term: 2 leader_uuid: "3e6a44464d7748fa8c784ef795b896fb" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.136466 5472 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "885de7b67f474f7bb54badfae193906d" candidate_uuid: "3e6a44464d7748fa8c784ef795b896fb" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
I20240501 08:13:06.136685 5472 raft_consensus.cc:3037] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.139315 5472 raft_consensus.cc:2454] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3e6a44464d7748fa8c784ef795b896fb in term 2.
I20240501 08:13:06.143254 8155 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.146067 3279 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.148298 3281 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.148317 6352 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.149833 6351 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.154685 5077 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.160774 3003 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.161123 5472 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.161162 3280 consensus_queue.cc:237] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.148873 8517 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.162868 8513 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.163826 5079 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.164402 8512 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.166754 8480 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.166837 3411 consensus_queue.cc:237] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:06.170030 6351 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.170136 5210 raft_consensus.cc:1261] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.170542 5077 raft_consensus.cc:1261] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.170670 8155 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.170800 8387 consensus_queue.cc:1035] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.170548 8480 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.171203 8155 consensus_queue.cc:1035] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.162693 8502 raft_consensus.cc:2258] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.172262 5210 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.173698 8303 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.174854 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d110b6fb2855454892ab5359ec3210b0 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.177466 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c3672fbf92d4b54a91554f88382a5ae with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.177726 6351 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.180987 3004 catalog_manager.cc:5484] T 1c3672fbf92d4b54a91554f88382a5ae P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.183588 3003 catalog_manager.cc:5484] T d110b6fb2855454892ab5359ec3210b0 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.186558 5077 raft_consensus.cc:2932] T 1c3672fbf92d4b54a91554f88382a5ae P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.188722 8402 ts_tablet_manager.cc:1888] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.186959 5079 raft_consensus.cc:2932] T d110b6fb2855454892ab5359ec3210b0 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:06.191220 8402 log.cc:1196] T f75a6080c937423f8ffb9f75b96da670 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/f75a6080c937423f8ffb9f75b96da670
I20240501 08:13:06.191339 3130 tablet_service.cc:1514] Processing DeleteTablet for tablet d110b6fb2855454892ab5359ec3210b0 with delete_type TABLET_DATA_TOMBSTONED (TS 466a9d70da1b4b1e8579e40447ce0c25 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:52880
I20240501 08:13:06.192202 5391 raft_consensus.cc:1050] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER f0af5c767c1b496cb1163870b60b066a to VOTER
I20240501 08:13:06.193027 5391 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:06.196442 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c3672fbf92d4b54a91554f88382a5ae with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:06.196764 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet f75a6080c937423f8ffb9f75b96da670 (table rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]) successfully deleted
I20240501 08:13:06.198012 5474 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.198112 3538 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.198637 8387 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.199183 8480 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.202154 5078 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:06.202749 8480 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:06.207686 8502 ts_tablet_manager.cc:1875] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.219785 8517 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.220489 5473 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.225260 3280 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.227479 8525 tablet_replica.cc:330] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:06.227887 8525 raft_consensus.cc:2229] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.228740 8525 raft_consensus.cc:2258] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.230288 8525 ts_tablet_manager.cc:1875] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.230242 6352 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.230696 3003 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.236991 8389 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.241837 3539 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.244081 3150 consensus_queue.cc:237] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:06.246865 5474 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.248003 8517 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.248849 8288 raft_consensus.cc:979] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: : Instructing follower f0af5c767c1b496cb1163870b60b066a to start an election
I20240501 08:13:06.249022 8288 raft_consensus.cc:1067] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Signalling peer f0af5c767c1b496cb1163870b60b066a to start an election
I20240501 08:13:06.248591 5474 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.250264 5077 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1"
dest_uuid: "f0af5c767c1b496cb1163870b60b066a"
from {username='slave'} at 127.28.1.132:58015
I20240501 08:13:06.250571 5077 raft_consensus.cc:482] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:06.250759 5077 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.253613 8502 ts_tablet_manager.cc:1888] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.253585 5077 raft_consensus.cc:504] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:06.257054 5473 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.256461 8512 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.259579 3538 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "f0af5c767c1b496cb1163870b60b066a" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:13:06.259799 3538 raft_consensus.cc:3032] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:06.259914 3538 raft_consensus.cc:729] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:13:06.261016 3538 consensus_queue.cc:260] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:06.262467 3538 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.263295 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a3fce8868ca74a028492122fc2762001 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.263343 6352 raft_consensus.cc:1261] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.264179 3007 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.264395 3003 catalog_manager.cc:5484] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.265412 8502 log.cc:1196] T a68f4cc374ad4656b1de788033ece98c P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/a68f4cc374ad4656b1de788033ece98c
I20240501 08:13:06.266202 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet a68f4cc374ad4656b1de788033ece98c (table rebalance_test_table_4 [id=bd799f3f8b8643c8baa95d64952187f6]) successfully deleted
I20240501 08:13:06.266606 8502 tablet_replica.cc:330] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: stopping tablet replica
I20240501 08:13:06.266795 8502 raft_consensus.cc:2229] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.267217 8502 raft_consensus.cc:2258] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.270171 8502 ts_tablet_manager.cc:1875] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.270730 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet a3fce8868ca74a028492122fc2762001 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:06.273517 5077 leader_election.cc:288] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [CANDIDATE]: Term 2 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 1771142e14884570b63c687e982902aa (127.28.1.132:43555), b1a5f0e8d89547a1ac983c6a5256eca8 (127.28.1.136:42523)
I20240501 08:13:06.275740 8525 ts_tablet_manager.cc:1888] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.280100 3538 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f0af5c767c1b496cb1163870b60b066a in term 2.
I20240501 08:13:06.281219 8525 log.cc:1196] T 1c3672fbf92d4b54a91554f88382a5ae P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:13:06.281208 3280 consensus_queue.cc:237] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } }
I20240501 08:13:06.278121 5079 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, f0af5c767c1b496cb1163870b60b066a (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.282217 8517 consensus_queue.cc:1035] T a3fce8868ca74a028492122fc2762001 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.283936 5473 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.284641 8389 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.287273 8525 tablet_replica.cc:330] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:06.287789 8387 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.288507 5473 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.292008 8525 raft_consensus.cc:2229] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.292273 8525 raft_consensus.cc:2258] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.292870 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 1c3672fbf92d4b54a91554f88382a5ae (table rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]) successfully deleted
I20240501 08:13:06.293537 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6da0f7473ef248f2ad4f5621f3fbd25e with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.299592 8525 ts_tablet_manager.cc:1875] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.300056 5079 raft_consensus.cc:1261] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:06.300276 6352 raft_consensus.cc:2932] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:06.300815 8389 consensus_queue.cc:1035] T 6da0f7473ef248f2ad4f5621f3fbd25e P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.301249 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "f0af5c767c1b496cb1163870b60b066a" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:13:06.302465 5079 raft_consensus.cc:2932] T 6da0f7473ef248f2ad4f5621f3fbd25e P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.305632 3007 catalog_manager.cc:5484] T 6da0f7473ef248f2ad4f5621f3fbd25e P b1a5f0e8d89547a1ac983c6a5256eca8 reported cstate change: config changed from index 8 to 9, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } }
I20240501 08:13:06.307704 5473 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "babe0d774b24482dab935eccb99f31c1" candidate_uuid: "f0af5c767c1b496cb1163870b60b066a" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8"
I20240501 08:13:06.307984 5473 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.309808 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet 6da0f7473ef248f2ad4f5621f3fbd25e with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:06.310160 8402 tablet_replica.cc:330] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:06.310366 8402 raft_consensus.cc:2229] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.310571 8402 raft_consensus.cc:2258] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.310729 5473 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f0af5c767c1b496cb1163870b60b066a in term 2.
I20240501 08:13:06.311501 8402 ts_tablet_manager.cc:1875] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.311535 8502 ts_tablet_manager.cc:1888] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.311591 3150 raft_consensus.cc:3037] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.311710 8502 log.cc:1196] T d110b6fb2855454892ab5359ec3210b0 P 466a9d70da1b4b1e8579e40447ce0c25: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/d110b6fb2855454892ab5359ec3210b0
I20240501 08:13:06.314448 3150 raft_consensus.cc:2454] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f0af5c767c1b496cb1163870b60b066a in term 2.
I20240501 08:13:06.314852 2995 catalog_manager.cc:4830] TS 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): tablet d110b6fb2855454892ab5359ec3210b0 (table rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]) successfully deleted
I20240501 08:13:06.315164 5014 leader_election.cc:302] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 466a9d70da1b4b1e8579e40447ce0c25, f0af5c767c1b496cb1163870b60b066a; no voters:
I20240501 08:13:06.315547 8247 raft_consensus.cc:2781] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:06.319382 8402 ts_tablet_manager.cc:1888] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.319394 8525 ts_tablet_manager.cc:1888] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:06.319516 8525 log.cc:1196] T a3fce8868ca74a028492122fc2762001 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/a3fce8868ca74a028492122fc2762001
I20240501 08:13:06.320601 8402 log.cc:1196] T 6da0f7473ef248f2ad4f5621f3fbd25e P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:13:06.322440 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet 6da0f7473ef248f2ad4f5621f3fbd25e (table rebalance_test_table_7 [id=8d789c500aae461a90218acc0a2c6097]) successfully deleted
I20240501 08:13:06.322671 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet a3fce8868ca74a028492122fc2762001 (table rebalance_test_table_3 [id=784d805456824b8e80d9dcf81c9f55be]) successfully deleted
I20240501 08:13:06.328109 8247 raft_consensus.cc:686] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 2 LEADER]: Becoming Leader. State: Replica: f0af5c767c1b496cb1163870b60b066a, State: Running, Role: LEADER
I20240501 08:13:06.328704 8247 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:06.332532 8536 ts_tablet_manager.cc:902] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
I20240501 08:13:06.332871 3005 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a reported cstate change: term changed from 1 to 2, leader changed from 1771142e14884570b63c687e982902aa (127.28.1.132) to f0af5c767c1b496cb1163870b60b066a (127.28.1.133). New cstate: current_term: 2 leader_uuid: "f0af5c767c1b496cb1163870b60b066a" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.341061 8536 tablet_copy_client.cc:308] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:44953
I20240501 08:13:06.341907 3430 tablet_copy_service.cc:140] P 254a7288f105497eb60d1709036a71a9: Received BeginTabletCopySession request for tablet 4adf393d330f45ba95fb50c850f20348 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:47885)
I20240501 08:13:06.342088 3430 tablet_copy_service.cc:161] P 254a7288f105497eb60d1709036a71a9: Beginning new tablet copy session on tablet 4adf393d330f45ba95fb50c850f20348 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:47885: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-4adf393d330f45ba95fb50c850f20348
I20240501 08:13:06.344353 3430 tablet_copy_source_session.cc:205] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:06.345682 8536 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4adf393d330f45ba95fb50c850f20348. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:06.351603 8536 tablet_copy_client.cc:783] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:06.351866 8536 tablet_copy_client.cc:647] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:06.353715 8541 tablet_copy_client.cc:1002] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:06.451826 3280 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420"
dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46116
I20240501 08:13:06.452165 3280 raft_consensus.cc:595] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:06.475925 3280 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:06.478266 5210 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.478466 3410 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.478869 8389 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:06.479249 3217 consensus_peers.cc:485] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bdad8a9a16414c5698f9fafb7272bdb1. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:06.479601 8389 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:06.482972 3150 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 3e6a44464d7748fa8c784ef795b896fb: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.484159 8235 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.492564 8389 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.496623 3410 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.498080 5210 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.500010 3005 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.519183 3538 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:06.521641 5079 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240501 08:13:06.521901 3477 consensus_peers.cc:485] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d56791555f6644a3bcee602299a85761. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:06.522543 3410 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.522518 8288 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:06.523586 8288 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:06.525746 8336 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.531313 3003 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:06.531558 5079 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.538594 3410 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.556504 3280 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:06.558804 5341 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.559226 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.559465 5391 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:06.559810 8387 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:06.560331 3217 consensus_peers.cc:485] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7fde726fbe8d48f685a4580ff6ae4033. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:06.562243 5473 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer 3e6a44464d7748fa8c784ef795b896fb: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.563057 8235 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.573026 8387 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.575382 3411 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.576784 5340 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.583213 3007 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.588047 8389 raft_consensus.cc:979] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: : Instructing follower 8151eaf288354d60b73b3c090dd271fd to start an election
I20240501 08:13:06.588233 8389 raft_consensus.cc:1067] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Signalling peer 8151eaf288354d60b73b3c090dd271fd to start an election
I20240501 08:13:06.589618 5210 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420"
dest_uuid: "8151eaf288354d60b73b3c090dd271fd"
from {username='slave'} at 127.28.1.130:41195
I20240501 08:13:06.589959 5210 raft_consensus.cc:482] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:06.590129 5210 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.594208 5210 raft_consensus.cc:504] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.596812 3410 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 3e6a44464d7748fa8c784ef795b896fb: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.597659 3150 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "8151eaf288354d60b73b3c090dd271fd" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "466a9d70da1b4b1e8579e40447ce0c25"
I20240501 08:13:06.597921 3150 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.600083 8235 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.600685 5210 leader_election.cc:288] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [CANDIDATE]: Term 2 election: Requested vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:13:06.601318 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "8151eaf288354d60b73b3c090dd271fd" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:13:06.601536 3280 raft_consensus.cc:3032] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:06.601619 3280 raft_consensus.cc:729] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:13:06.601961 3280 consensus_queue.cc:260] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.602389 3280 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.607712 3280 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8151eaf288354d60b73b3c090dd271fd in term 2.
I20240501 08:13:06.610080 3150 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8151eaf288354d60b73b3c090dd271fd in term 2.
I20240501 08:13:06.610613 5146 leader_election.cc:302] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 466a9d70da1b4b1e8579e40447ce0c25, 8151eaf288354d60b73b3c090dd271fd, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:13:06.611064 8256 raft_consensus.cc:2781] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:06.627297 8256 raft_consensus.cc:686] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 2 LEADER]: Becoming Leader. State: Replica: 8151eaf288354d60b73b3c090dd271fd, State: Running, Role: LEADER
I20240501 08:13:06.628355 8256 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:06.651991 3007 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd reported cstate change: term changed from 1 to 2, leader changed from e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) to 8151eaf288354d60b73b3c090dd271fd (127.28.1.134). New cstate: current_term: 2 leader_uuid: "8151eaf288354d60b73b3c090dd271fd" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.654228 5341 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4cb5c6a514354708bff63dd5e51c7420" candidate_uuid: "8151eaf288354d60b73b3c090dd271fd" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "3e6a44464d7748fa8c784ef795b896fb"
I20240501 08:13:06.654476 5341 raft_consensus.cc:3037] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:06.659435 5341 raft_consensus.cc:2454] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8151eaf288354d60b73b3c090dd271fd in term 2.
I20240501 08:13:06.683074 3149 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:06.711262 5341 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.712038 8517 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:13:06.713078 3086 consensus_peers.cc:485] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 -> Peer 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431): Couldn't send request to peer 53ae6b4d587b4c48a6dcc10a7ba67579. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e62d72ab5f9b460aaa7fe7bb7ecd7b08. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:06.713407 3410 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:13:06.715109 8513 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:06.737780 8517 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.738406 3410 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.743070 5341 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.746379 3007 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 6 to 7, NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } } }
I20240501 08:13:06.757588 5340 consensus_queue.cc:237] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:06.760038 3149 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 3e6a44464d7748fa8c784ef795b896fb: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:06.760016 5473 raft_consensus.cc:1261] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer 3e6a44464d7748fa8c784ef795b896fb: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:06.760557 8635 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:06.761368 8235 consensus_queue.cc:1035] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:06.763767 8630 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:06.764379 5473 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:06.767659 3149 raft_consensus.cc:2932] T 885de7b67f474f7bb54badfae193906d P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } }
I20240501 08:13:06.768556 2994 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 885de7b67f474f7bb54badfae193906d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:06.772282 3003 catalog_manager.cc:5484] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 8 to 10, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 2 leader_uuid: "3e6a44464d7748fa8c784ef795b896fb" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:06.776300 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet 885de7b67f474f7bb54badfae193906d with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:06.777999 8652 tablet_replica.cc:330] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:06.778218 8652 raft_consensus.cc:2229] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:06.778439 8652 raft_consensus.cc:2258] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:06.780107 8652 ts_tablet_manager.cc:1875] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:06.780208 3538 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Refusing update from remote peer f0af5c767c1b496cb1163870b60b066a: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.780895 8247 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.788967 8652 ts_tablet_manager.cc:1888] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:06.789119 8652 log.cc:1196] T 885de7b67f474f7bb54badfae193906d P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/885de7b67f474f7bb54badfae193906d
I20240501 08:13:06.790076 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet 885de7b67f474f7bb54badfae193906d (table rebalance_test_table_1 [id=5b445f86623f402bad859a1c5dc4e359]) successfully deleted
I20240501 08:13:06.791606 5473 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer f0af5c767c1b496cb1163870b60b066a: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.792332 8247 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.800096 3149 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer f0af5c767c1b496cb1163870b60b066a: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:06.800921 8537 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:06.947281 8734 ts_tablet_manager.cc:902] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:06.948565 8734 tablet_copy_client.cc:308] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:06.949198 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet 7fde726fbe8d48f685a4580ff6ae4033 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:34297)
I20240501 08:13:06.949363 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet 7fde726fbe8d48f685a4580ff6ae4033 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:34297: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:13:06.951282 3301 tablet_copy_source_session.cc:205] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:06.952539 8734 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fde726fbe8d48f685a4580ff6ae4033. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:06.956699 8734 tablet_copy_client.cc:783] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:06.956944 8734 tablet_copy_client.cc:647] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:06.958438 8738 tablet_copy_client.cc:1002] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:07.069506 8739 ts_tablet_manager.cc:902] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 1771142e14884570b63c687e982902aa (127.28.1.132:43555)
I20240501 08:13:07.071420 8743 ts_tablet_manager.cc:902] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531)
I20240501 08:13:07.072350 8739 tablet_copy_client.cc:308] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43555
I20240501 08:13:07.073184 3560 tablet_copy_service.cc:140] P 1771142e14884570b63c687e982902aa: Received BeginTabletCopySession request for tablet d56791555f6644a3bcee602299a85761 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:43071)
I20240501 08:13:07.073365 3560 tablet_copy_service.cc:161] P 1771142e14884570b63c687e982902aa: Beginning new tablet copy session on tablet d56791555f6644a3bcee602299a85761 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:43071: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-d56791555f6644a3bcee602299a85761
I20240501 08:13:07.073558 8743 tablet_copy_client.cc:308] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.130:40531
I20240501 08:13:07.074244 3301 tablet_copy_service.cc:140] P e3b81e5e25de4eddbeaad1e5834c0836: Received BeginTabletCopySession request for tablet bdad8a9a16414c5698f9fafb7272bdb1 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:34297)
I20240501 08:13:07.074376 3301 tablet_copy_service.cc:161] P e3b81e5e25de4eddbeaad1e5834c0836: Beginning new tablet copy session on tablet bdad8a9a16414c5698f9fafb7272bdb1 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:34297: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:13:07.075546 3560 tablet_copy_source_session.cc:205] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:07.076566 3301 tablet_copy_source_session.cc:205] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:07.076879 8739 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d56791555f6644a3bcee602299a85761. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:07.078042 8743 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bdad8a9a16414c5698f9fafb7272bdb1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:07.079030 3149 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 8151eaf288354d60b73b3c090dd271fd: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:07.079941 8256 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:07.083204 8739 tablet_copy_client.cc:783] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:07.083456 8739 tablet_copy_client.cc:647] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:07.085109 8743 tablet_copy_client.cc:783] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:07.085175 8744 tablet_copy_client.cc:1002] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:07.085338 8743 tablet_copy_client.cc:647] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:07.086720 8748 tablet_copy_client.cc:1002] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:07.115671 5341 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 8151eaf288354d60b73b3c090dd271fd: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:07.116394 8256 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:07.126363 3280 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Refusing update from remote peer 8151eaf288354d60b73b3c090dd271fd: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:07.127172 8629 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:07.137652 5210 consensus_queue.cc:237] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:07.139534 3149 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer 8151eaf288354d60b73b3c090dd271fd: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:07.139712 5341 raft_consensus.cc:1261] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 8151eaf288354d60b73b3c090dd271fd: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:07.140215 8750 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:07.140496 8751 consensus_queue.cc:1035] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:07.142730 8751 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:07.143730 5341 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:07.147063 3149 raft_consensus.cc:2932] T 4cb5c6a514354708bff63dd5e51c7420 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } }
I20240501 08:13:07.148269 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4cb5c6a514354708bff63dd5e51c7420 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:07.148949 3005 catalog_manager.cc:5484] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd reported cstate change: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 2 leader_uuid: "8151eaf288354d60b73b3c090dd271fd" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:07.153059 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 4cb5c6a514354708bff63dd5e51c7420 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:07.157097 8755 tablet_replica.cc:330] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:07.158568 8755 raft_consensus.cc:2229] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:07.159353 8755 raft_consensus.cc:2258] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:07.160357 8755 ts_tablet_manager.cc:1875] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:07.165895 8755 ts_tablet_manager.cc:1888] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:07.166024 8755 log.cc:1196] T 4cb5c6a514354708bff63dd5e51c7420 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/4cb5c6a514354708bff63dd5e51c7420
I20240501 08:13:07.166931 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 4cb5c6a514354708bff63dd5e51c7420 (table rebalance_test_table_2 [id=873554f28bb94ceeacc5062612792e4d]) successfully deleted
I20240501 08:13:07.233319 8790 ts_tablet_manager.cc:902] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Initiating tablet copy from peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945)
I20240501 08:13:07.234956 5079 consensus_queue.cc:237] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:07.237017 5474 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Refusing update from remote peer f0af5c767c1b496cb1163870b60b066a: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:07.237022 3149 raft_consensus.cc:1261] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Refusing update from remote peer f0af5c767c1b496cb1163870b60b066a: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:07.237638 8660 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:07.238310 8660 consensus_queue.cc:1035] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [LEADER]: Connected to new peer: Peer: permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:07.240542 8247 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:07.241668 3149 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P 466a9d70da1b4b1e8579e40447ce0c25 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:07.243729 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet babe0d774b24482dab935eccb99f31c1 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:07.244320 8790 tablet_copy_client.cc:308] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:42945
I20240501 08:13:07.245054 3170 tablet_copy_service.cc:140] P 466a9d70da1b4b1e8579e40447ce0c25: Received BeginTabletCopySession request for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 ({username='slave'} at 127.28.1.137:49567)
I20240501 08:13:07.245193 3170 tablet_copy_service.cc:161] P 466a9d70da1b4b1e8579e40447ce0c25: Beginning new tablet copy session on tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 from peer 53ae6b4d587b4c48a6dcc10a7ba67579 at {username='slave'} at 127.28.1.137:49567: session id = 53ae6b4d587b4c48a6dcc10a7ba67579-e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:13:07.244997 5474 raft_consensus.cc:2932] T babe0d774b24482dab935eccb99f31c1 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } }
I20240501 08:13:07.245936 3007 catalog_manager.cc:5484] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a reported cstate change: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 2 leader_uuid: "f0af5c767c1b496cb1163870b60b066a" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:07.248080 3170 tablet_copy_source_session.cc:205] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:07.249541 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet babe0d774b24482dab935eccb99f31c1 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:07.250039 8790 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:07.250561 8799 tablet_replica.cc:330] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:07.250833 8799 raft_consensus.cc:2229] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:07.251091 8799 raft_consensus.cc:2258] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:07.252390 8799 ts_tablet_manager.cc:1875] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:07.255203 8790 tablet_copy_client.cc:783] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:07.255432 8790 tablet_copy_client.cc:647] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:07.257213 8798 tablet_copy_client.cc:1002] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Injecting latency into file download: 1500
I20240501 08:13:07.259649 8799 ts_tablet_manager.cc:1888] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:07.259775 8799 log.cc:1196] T babe0d774b24482dab935eccb99f31c1 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/babe0d774b24482dab935eccb99f31c1
I20240501 08:13:07.260650 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet babe0d774b24482dab935eccb99f31c1 (table rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]) successfully deleted
I20240501 08:13:07.854107 8536 tablet_copy_client.cc:523] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:07.857457 8536 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:07.898180 8536 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:07.898553 8536 tablet_bootstrap.cc:492] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:07.898864 8536 ts_tablet_manager.cc:1367] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.041s user 0.039s sys 0.000s
I20240501 08:13:07.899679 8536 raft_consensus.cc:348] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:07.899905 8536 raft_consensus.cc:729] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:07.900137 8536 consensus_queue.cc:260] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:07.900909 8536 ts_tablet_manager.cc:1398] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240501 08:13:07.901577 3430 tablet_copy_service.cc:342] P 254a7288f105497eb60d1709036a71a9: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-4adf393d330f45ba95fb50c850f20348 received from {username='slave'} at 127.28.1.137:47885
I20240501 08:13:07.901727 3430 tablet_copy_service.cc:433] P 254a7288f105497eb60d1709036a71a9: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-4adf393d330f45ba95fb50c850f20348 on tablet 4adf393d330f45ba95fb50c850f20348 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:08.082983 6351 raft_consensus.cc:1203] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:08.458874 8734 tablet_copy_client.cc:523] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:08.461623 8734 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:08.491657 8734 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:08.492074 8734 tablet_bootstrap.cc:492] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:08.492339 8734 ts_tablet_manager.cc:1367] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.031s user 0.031s sys 0.001s
I20240501 08:13:08.493003 8734 raft_consensus.cc:348] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.493220 8734 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:08.493439 8734 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.493934 8734 ts_tablet_manager.cc:1398] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240501 08:13:08.494722 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-7fde726fbe8d48f685a4580ff6ae4033 received from {username='slave'} at 127.28.1.137:34297
I20240501 08:13:08.494887 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-7fde726fbe8d48f685a4580ff6ae4033 on tablet 7fde726fbe8d48f685a4580ff6ae4033 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:08.507551 6351 raft_consensus.cc:1203] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:08.585534 8739 tablet_copy_client.cc:523] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:08.587049 8743 tablet_copy_client.cc:523] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:08.589226 8739 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:08.589572 8743 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:08.602262 8816 raft_consensus.cc:1050] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:08.602969 8816 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:08.605365 6351 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.606122 5077 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.606913 8969 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.607439 8969 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.613211 3280 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.613919 8971 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.623176 8969 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.623945 3280 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.624256 6352 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.629119 3005 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:08.629314 5077 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.649304 8743 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:08.649824 8743 tablet_bootstrap.cc:492] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:08.650144 8743 ts_tablet_manager.cc:1367] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.061s user 0.047s sys 0.004s
I20240501 08:13:08.651050 8743 raft_consensus.cc:348] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.651340 8743 raft_consensus.cc:729] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:08.651556 8743 consensus_queue.cc:260] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.652302 8743 ts_tablet_manager.cc:1398] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:13:08.653177 3301 tablet_copy_service.cc:342] P e3b81e5e25de4eddbeaad1e5834c0836: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-bdad8a9a16414c5698f9fafb7272bdb1 received from {username='slave'} at 127.28.1.137:34297
I20240501 08:13:08.653327 3301 tablet_copy_service.cc:433] P e3b81e5e25de4eddbeaad1e5834c0836: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-bdad8a9a16414c5698f9fafb7272bdb1 on tablet bdad8a9a16414c5698f9fafb7272bdb1 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:08.656626 8739 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:08.657088 8739 tablet_bootstrap.cc:492] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:08.657384 8739 ts_tablet_manager.cc:1367] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.068s user 0.042s sys 0.010s
I20240501 08:13:08.658228 8739 raft_consensus.cc:348] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.658509 8739 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:08.658715 8739 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.659401 8739 ts_tablet_manager.cc:1398] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:13:08.660265 3560 tablet_copy_service.cc:342] P 1771142e14884570b63c687e982902aa: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-d56791555f6644a3bcee602299a85761 received from {username='slave'} at 127.28.1.137:43071
I20240501 08:13:08.660396 3560 tablet_copy_service.cc:433] P 1771142e14884570b63c687e982902aa: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-d56791555f6644a3bcee602299a85761 on tablet d56791555f6644a3bcee602299a85761 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:08.669351 3410 consensus_queue.cc:237] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:08.672008 5077 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:08.672008 6352 raft_consensus.cc:1261] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer 254a7288f105497eb60d1709036a71a9: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:08.672551 8969 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:08.672863 8971 consensus_queue.cc:1035] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:08.675043 8969 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.675688 5077 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.676039 6352 raft_consensus.cc:2932] T 4adf393d330f45ba95fb50c850f20348 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.679380 2993 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4adf393d330f45ba95fb50c850f20348 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:08.680724 3003 catalog_manager.cc:5484] T 4adf393d330f45ba95fb50c850f20348 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 8 to 9, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "254a7288f105497eb60d1709036a71a9" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:08.685060 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 4adf393d330f45ba95fb50c850f20348 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:08.685768 8986 tablet_replica.cc:330] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:08.685974 8986 raft_consensus.cc:2229] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:08.686408 8986 raft_consensus.cc:2258] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:08.687602 8986 ts_tablet_manager.cc:1875] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:08.693254 8986 ts_tablet_manager.cc:1888] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:08.693488 8986 log.cc:1196] T 4adf393d330f45ba95fb50c850f20348 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/4adf393d330f45ba95fb50c850f20348
I20240501 08:13:08.694303 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 4adf393d330f45ba95fb50c850f20348 (table rebalance_test_table_6 [id=916ddf78fb2a434e83baed80751cb4f8]) successfully deleted
I20240501 08:13:08.757589 8790 tablet_copy_client.cc:523] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:08.761301 8790 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap starting.
I20240501 08:13:08.795173 8790 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:08.795529 8790 tablet_bootstrap.cc:492] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Bootstrap complete.
I20240501 08:13:08.795773 8790 ts_tablet_manager.cc:1367] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent bootstrapping tablet: real 0.035s user 0.023s sys 0.007s
I20240501 08:13:08.796479 8790 raft_consensus.cc:348] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.796695 8790 raft_consensus.cc:729] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Initialized, Role: LEARNER
I20240501 08:13:08.796873 8790 consensus_queue.cc:260] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: true } }
I20240501 08:13:08.797308 8790 ts_tablet_manager.cc:1398] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579: Time spent starting tablet: real 0.001s user 0.000s sys 0.004s
I20240501 08:13:08.798046 3170 tablet_copy_service.cc:342] P 466a9d70da1b4b1e8579e40447ce0c25: Request end of tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-e62d72ab5f9b460aaa7fe7bb7ecd7b08 received from {username='slave'} at 127.28.1.137:49567
I20240501 08:13:08.798182 3170 tablet_copy_service.cc:433] P 466a9d70da1b4b1e8579e40447ce0c25: ending tablet copy session 53ae6b4d587b4c48a6dcc10a7ba67579-e62d72ab5f9b460aaa7fe7bb7ecd7b08 on tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with peer 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:08.964548 9026 raft_consensus.cc:1050] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:08.965193 9026 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:08.968080 5341 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.968138 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.968633 8854 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.969013 6352 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:08.969444 8854 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.969715 9026 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:08.973090 9026 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.973902 3410 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.977768 5341 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.979997 3003 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:08.990926 6352 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.016757 3280 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033"
dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46362
I20240501 08:13:09.016988 3280 raft_consensus.cc:595] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:09.044418 6352 raft_consensus.cc:1203] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:09.064347 6352 raft_consensus.cc:1203] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:09.268112 3280 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033"
dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46394
I20240501 08:13:09.268298 3280 raft_consensus.cc:595] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:09.294387 6352 raft_consensus.cc:1203] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240501 08:13:09.439074 8854 raft_consensus.cc:979] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: : Instructing follower 53ae6b4d587b4c48a6dcc10a7ba67579 to start an election
I20240501 08:13:09.439239 8854 raft_consensus.cc:1067] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Signalling peer 53ae6b4d587b4c48a6dcc10a7ba67579 to start an election
I20240501 08:13:09.440073 6352 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033"
dest_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579"
from {username='slave'} at 127.28.1.130:43831
I20240501 08:13:09.440369 6352 raft_consensus.cc:482] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:09.440572 6352 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.442617 6352 raft_consensus.cc:504] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.444180 6352 leader_election.cc:288] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 election: Requested vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:13:09.444672 3410 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:13:09.444923 3410 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.445086 3280 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "e3b81e5e25de4eddbeaad1e5834c0836"
I20240501 08:13:09.445289 3280 raft_consensus.cc:3032] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:09.445397 3280 raft_consensus.cc:729] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: e3b81e5e25de4eddbeaad1e5834c0836, State: Running, Role: LEADER
I20240501 08:13:09.445647 3280 consensus_queue.cc:260] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.446111 3280 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.447530 3410 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:09.448611 3280 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:09.449074 6284 leader_election.cc:302] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 254a7288f105497eb60d1709036a71a9, 53ae6b4d587b4c48a6dcc10a7ba67579, e3b81e5e25de4eddbeaad1e5834c0836; no voters:
I20240501 08:13:09.449476 8897 raft_consensus.cc:2781] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:09.449975 8897 raft_consensus.cc:686] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 LEADER]: Becoming Leader. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Running, Role: LEADER
I20240501 08:13:09.450497 8897 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.454409 3005 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: term changed from 1 to 2, leader changed from e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) to 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137). New cstate: current_term: 2 leader_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:09.454808 5340 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fde726fbe8d48f685a4580ff6ae4033" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "3e6a44464d7748fa8c784ef795b896fb"
I20240501 08:13:09.455004 5340 raft_consensus.cc:3037] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.457298 5340 raft_consensus.cc:2454] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:09.470077 9032 raft_consensus.cc:1050] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:09.470808 9032 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.473517 6352 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.473773 5077 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.473796 3410 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 1771142e14884570b63c687e982902aa: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.474385 9032 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.474809 8908 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.475333 9087 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.478024 8908 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.479045 6351 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.479025 5077 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.484220 8854 raft_consensus.cc:1050] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:09.484194 3004 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "1771142e14884570b63c687e982902aa" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.484166 3410 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.485042 8854 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.500309 3410 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.500488 6351 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.499891 5210 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.500864 9026 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.501199 8389 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.501466 8854 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.505465 9026 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.505563 5210 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.508481 3410 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.510459 6351 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.511161 3003 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:09.520275 3280 consensus_queue.cc:237] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.522230 6351 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:09.522239 5210 raft_consensus.cc:1261] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Refusing update from remote peer e3b81e5e25de4eddbeaad1e5834c0836: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:09.522823 8389 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.523125 8854 consensus_queue.cc:1035] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.525665 8854 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.526748 6351 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.526991 5210 raft_consensus.cc:2932] T bdad8a9a16414c5698f9fafb7272bdb1 P 8151eaf288354d60b73b3c090dd271fd [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.530292 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bdad8a9a16414c5698f9fafb7272bdb1 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:09.531668 3004 catalog_manager.cc:5484] T bdad8a9a16414c5698f9fafb7272bdb1 P e3b81e5e25de4eddbeaad1e5834c0836 reported cstate change: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:09.535845 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet bdad8a9a16414c5698f9fafb7272bdb1 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:09.537196 9106 tablet_replica.cc:330] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:09.537555 9106 raft_consensus.cc:2229] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:09.537950 9106 raft_consensus.cc:2258] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:09.538728 9106 ts_tablet_manager.cc:1875] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:09.543990 9106 ts_tablet_manager.cc:1888] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:09.544102 9106 log.cc:1196] T bdad8a9a16414c5698f9fafb7272bdb1 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:13:09.544852 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet bdad8a9a16414c5698f9fafb7272bdb1 (table rebalance_test_table_9 [id=af1c0f24a9cb4640b10457a508597f51]) successfully deleted
I20240501 08:13:09.554483 3538 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d56791555f6644a3bcee602299a85761"
dest_uuid: "1771142e14884570b63c687e982902aa"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:59112
I20240501 08:13:09.554667 3538 raft_consensus.cc:595] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:09.607034 3538 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d56791555f6644a3bcee602299a85761"
dest_uuid: "1771142e14884570b63c687e982902aa"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:59140
I20240501 08:13:09.607244 3538 raft_consensus.cc:595] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:09.695744 9143 raft_consensus.cc:1050] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25: attempting to promote NON_VOTER 53ae6b4d587b4c48a6dcc10a7ba67579 to VOTER
I20240501 08:13:09.696666 9143 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.699813 5340 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.699841 6351 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 LEARNER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.700039 3410 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:13:09.700415 9142 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.700830 9145 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.701160 9143 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:13:09.709170 9145 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.709904 3410 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.712864 6351 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.718310 3005 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 reported cstate change: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:09.721581 5340 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } attrs { replace: true } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.726455 3149 consensus_queue.cc:237] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.728300 6351 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:09.728797 8938 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.730118 5340 raft_consensus.cc:1261] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Refusing update from remote peer 466a9d70da1b4b1e8579e40447ce0c25: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:13:09.730576 8938 consensus_queue.cc:1035] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.731225 9143 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 466a9d70da1b4b1e8579e40447ce0c25 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.731748 6351 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.736058 2995 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:09.736752 3001 catalog_manager.cc:5484] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "466a9d70da1b4b1e8579e40447ce0c25" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.737285 5340 raft_consensus.cc:2932] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 254a7288f105497eb60d1709036a71a9 (127.28.1.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.743237 3390 tablet_service.cc:1514] Processing DeleteTablet for tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 with delete_type TABLET_DATA_TOMBSTONED (TS 254a7288f105497eb60d1709036a71a9 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56990
I20240501 08:13:09.743629 9106 tablet_replica.cc:330] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: stopping tablet replica
I20240501 08:13:09.743862 9106 raft_consensus.cc:2229] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:09.744167 9106 raft_consensus.cc:2258] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:09.745059 9106 ts_tablet_manager.cc:1875] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:09.749209 9106 ts_tablet_manager.cc:1888] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240501 08:13:09.749329 9106 log.cc:1196] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 254a7288f105497eb60d1709036a71a9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:13:09.749943 2993 catalog_manager.cc:4830] TS 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08 (table rebalance_test_table_0 [id=e4a66f6b14974fc2886b51df4f7e81bd]) successfully deleted
I20240501 08:13:09.829849 5340 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:09.830597 8897 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.836133 3410 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:09.836666 8897 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.842814 3280 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:09.843906 8897 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:09.859174 6351 consensus_queue.cc:237] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.861855 3411 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:09.862802 9167 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:09.863195 5340 raft_consensus.cc:1261] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:09.863741 9167 consensus_queue.cc:1035] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:09.865128 9085 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.866540 5340 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.868399 3411 raft_consensus.cc:2932] T 7fde726fbe8d48f685a4580ff6ae4033 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.869240 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7fde726fbe8d48f685a4580ff6ae4033 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:09.872040 3005 catalog_manager.cc:5484] T 7fde726fbe8d48f685a4580ff6ae4033 P 3e6a44464d7748fa8c784ef795b896fb reported cstate change: config changed from index 8 to 10, VOTER e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130) evicted. New cstate: current_term: 2 leader_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:09.875752 3259 tablet_service.cc:1514] Processing DeleteTablet for tablet 7fde726fbe8d48f685a4580ff6ae4033 with delete_type TABLET_DATA_TOMBSTONED (TS e3b81e5e25de4eddbeaad1e5834c0836 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:47226
I20240501 08:13:09.876513 9175 tablet_replica.cc:330] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: stopping tablet replica
I20240501 08:13:09.876730 9175 raft_consensus.cc:2229] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:09.876915 9175 raft_consensus.cc:2258] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:09.877665 9175 ts_tablet_manager.cc:1875] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:09.881603 3538 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "d56791555f6644a3bcee602299a85761"
dest_uuid: "1771142e14884570b63c687e982902aa"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:59156
I20240501 08:13:09.881793 3538 raft_consensus.cc:595] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:09.884104 9175 ts_tablet_manager.cc:1888] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:09.884285 9175 log.cc:1196] T 7fde726fbe8d48f685a4580ff6ae4033 P e3b81e5e25de4eddbeaad1e5834c0836: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:13:09.884961 2996 catalog_manager.cc:4830] TS e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): tablet 7fde726fbe8d48f685a4580ff6ae4033 (table rebalance_test_table_5 [id=767402bf16774ccea2fcb1898a3790bf]) successfully deleted
I20240501 08:13:09.891753 9034 raft_consensus.cc:979] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: : Instructing follower 53ae6b4d587b4c48a6dcc10a7ba67579 to start an election
I20240501 08:13:09.891947 9034 raft_consensus.cc:1067] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Signalling peer 53ae6b4d587b4c48a6dcc10a7ba67579 to start an election
I20240501 08:13:09.892774 6351 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "d56791555f6644a3bcee602299a85761"
dest_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579"
from {username='slave'} at 127.28.1.132:58119
I20240501 08:13:09.892963 6351 raft_consensus.cc:482] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:09.893080 6351 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.895339 6351 raft_consensus.cc:504] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.896205 6351 leader_election.cc:288] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 election: Requested vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953), f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351)
I20240501 08:13:09.896517 3538 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "1771142e14884570b63c687e982902aa"
I20240501 08:13:09.896709 3538 raft_consensus.cc:3032] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:09.896800 3538 raft_consensus.cc:729] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 1771142e14884570b63c687e982902aa, State: Running, Role: LEADER
I20240501 08:13:09.896886 3411 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "254a7288f105497eb60d1709036a71a9"
I20240501 08:13:09.897097 3411 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.897035 3538 consensus_queue.cc:260] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.897382 3538 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.898936 3411 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:09.899159 3538 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:09.899809 6284 leader_election.cc:302] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 1771142e14884570b63c687e982902aa, 254a7288f105497eb60d1709036a71a9, 53ae6b4d587b4c48a6dcc10a7ba67579; no voters:
I20240501 08:13:09.900180 8897 raft_consensus.cc:2781] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:09.900324 8897 raft_consensus.cc:686] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 LEADER]: Becoming Leader. State: Replica: 53ae6b4d587b4c48a6dcc10a7ba67579, State: Running, Role: LEADER
I20240501 08:13:09.900617 8897 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:09.903810 3004 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: term changed from 1 to 2, leader changed from 1771142e14884570b63c687e982902aa (127.28.1.132) to 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137). New cstate: current_term: 2 leader_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:09.905584 5077 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d56791555f6644a3bcee602299a85761" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "f0af5c767c1b496cb1163870b60b066a"
I20240501 08:13:09.905921 5077 raft_consensus.cc:3037] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:09.909464 5077 raft_consensus.cc:2454] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 2.
I20240501 08:13:10.395705 3538 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:10.396262 8897 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:10.407066 5077 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:10.407608 8897 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:10.411626 3411 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240501 08:13:10.412222 8897 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:13:11.014843 6351 consensus_queue.cc:237] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:11.016742 5077 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:11.016933 3411 raft_consensus.cc:1261] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Refusing update from remote peer 53ae6b4d587b4c48a6dcc10a7ba67579: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240501 08:13:11.017316 9226 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:11.017978 9227 consensus_queue.cc:1035] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [LEADER]: Connected to new peer: Peer: permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:13:11.019730 9226 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:11.020243 5077 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P f0af5c767c1b496cb1163870b60b066a [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:11.022753 2996 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d56791555f6644a3bcee602299a85761 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:11.023103 3411 raft_consensus.cc:2932] T d56791555f6644a3bcee602299a85761 P 254a7288f105497eb60d1709036a71a9 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } } }
I20240501 08:13:11.027622 3005 catalog_manager.cc:5484] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 reported cstate change: config changed from index 8 to 10, VOTER 1771142e14884570b63c687e982902aa (127.28.1.132) evicted. New cstate: current_term: 2 leader_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f0af5c767c1b496cb1163870b60b066a" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 43351 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:11.032896 3520 tablet_service.cc:1514] Processing DeleteTablet for tablet d56791555f6644a3bcee602299a85761 with delete_type TABLET_DATA_TOMBSTONED (TS 1771142e14884570b63c687e982902aa not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:35494
I20240501 08:13:11.034912 9233 tablet_replica.cc:330] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: stopping tablet replica
I20240501 08:13:11.035161 9233 raft_consensus.cc:2229] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:11.035457 9233 raft_consensus.cc:2258] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:11.036618 9233 ts_tablet_manager.cc:1875] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:11.041854 9233 ts_tablet_manager.cc:1888] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240501 08:13:11.041993 9233 log.cc:1196] T d56791555f6644a3bcee602299a85761 P 1771142e14884570b63c687e982902aa: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/d56791555f6644a3bcee602299a85761
I20240501 08:13:11.042730 2996 catalog_manager.cc:4830] TS 1771142e14884570b63c687e982902aa (127.28.1.132:43555): tablet d56791555f6644a3bcee602299a85761 (table rebalance_test_table_8 [id=a5903000b529447a8516c8ff162a81ff]) successfully deleted
I20240501 08:13:11.301520 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:11.318691 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:11.324234 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:11.341009 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:11.354030 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:11.365056 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:11.406687 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:11.433023 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:11.455924 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:11.637200 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:11.658924 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:11.666191 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:11.693540 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:11.702215 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:11.705010 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:11.732180 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:11.744930 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:11.759814 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.577785 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:12.595240 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:12.595466 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:12.601989 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.640398 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:12.666976 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.695904 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:12.698094 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.721838 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:12.938694 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:12.939666 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:12.941473 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.944029 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:12.963754 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:12.977645 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:12.999701 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:13.000438 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:13.062070 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.307948 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:13.314255 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:13.324148 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:13.326057 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.343848 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:13.363605 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:13.367019 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:13.379285 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.385186 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.736413 3520 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240501 08:13:13.739224 3390 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:13:13.766471 5322 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.769678 3130 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:13.781647 6332 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240501 08:13:13.798377 5191 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:13.806398 5454 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:13.818753 3259 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240501 08:13:13.820417 5059 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
975c4e11a5ae4beabe81775b2a092a40 | 127.28.1.190:38185 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:37195 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
1771142e14884570b63c687e982902aa | 127.28.1.132:43555 | HEALTHY | <none> | 8 | 0
254a7288f105497eb60d1709036a71a9 | 127.28.1.131:44953 | HEALTHY | <none> | 3 | 0
3e6a44464d7748fa8c784ef795b896fb | 127.28.1.135:42375 | HEALTHY | <none> | 1 | 0
466a9d70da1b4b1e8579e40447ce0c25 | 127.28.1.129:42945 | HEALTHY | <none> | 5 | 0
53ae6b4d587b4c48a6dcc10a7ba67579 | 127.28.1.137:42431 | HEALTHY | <none> | 2 | 0
8151eaf288354d60b73b3c090dd271fd | 127.28.1.134:44365 | HEALTHY | <none> | 1 | 0
b1a5f0e8d89547a1ac983c6a5256eca8 | 127.28.1.136:42523 | HEALTHY | <none> | 0 | 0
e3b81e5e25de4eddbeaad1e5834c0836 | 127.28.1.130:40531 | HEALTHY | <none> | 9 | 0
f0af5c767c1b496cb1163870b60b066a | 127.28.1.133:43351 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 9
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
---------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 9 server(s) checked
local_ip_for_outbound_sockets | 127.28.1.129 | experimental | 127.28.1.129:42945
local_ip_for_outbound_sockets | 127.28.1.130 | experimental | 127.28.1.130:40531
local_ip_for_outbound_sockets | 127.28.1.131 | experimental | 127.28.1.131:44953
local_ip_for_outbound_sockets | 127.28.1.132 | experimental | 127.28.1.132:43555
local_ip_for_outbound_sockets | 127.28.1.133 | experimental | 127.28.1.133:43351
local_ip_for_outbound_sockets | 127.28.1.134 | experimental | 127.28.1.134:44365
local_ip_for_outbound_sockets | 127.28.1.135 | experimental | 127.28.1.135:42375
local_ip_for_outbound_sockets | 127.28.1.136 | experimental | 127.28.1.136:42523
local_ip_for_outbound_sockets | 127.28.1.137 | experimental | 127.28.1.137:42431
never_fsync | true | unsafe,advanced | all 9 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 9 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 9 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 9 server(s) checked
server_dump_info_format | pb | hidden | all 9 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.28.1.129:42945
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.28.1.130:40531
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.28.1.131:44953
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.28.1.132:43555
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.28.1.133:43351
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.28.1.134:44365
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.28.1.135:42375
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.28.1.136:42523
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb | hidden | 127.28.1.137:42431
tablet_copy_download_file_inject_latency_ms | 1500 | hidden | all 9 server(s) checked
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:37195 | all 9 server(s) checked
time_source | builtin | all 9 server(s) checked
Version Summary
Version | Servers
-----------------+--------------------------
1.18.0-SNAPSHOT | all 10 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
rebalance_test_table_0 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_5 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_6 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_7 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_8 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_9 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 9
Tables | 10
Tablets | 30
Replicas | 90
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240501 08:13:13.906939 28678 log_verifier.cc:126] Checking tablet 0434084d13f740368d1daae53dee9fe2
I20240501 08:13:13.947266 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 0434084d13f740368d1daae53dee9fe2
I20240501 08:13:13.947379 28678 log_verifier.cc:126] Checking tablet 0612f1f8e4ca4a29971151bf52571a33
I20240501 08:13:13.984572 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 0612f1f8e4ca4a29971151bf52571a33
I20240501 08:13:13.984673 28678 log_verifier.cc:126] Checking tablet 0bd2ea6b1e714d619d7dc00d20067a59
I20240501 08:13:14.020257 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 0bd2ea6b1e714d619d7dc00d20067a59
I20240501 08:13:14.020367 28678 log_verifier.cc:126] Checking tablet 1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:13:14.058842 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 1c3672fbf92d4b54a91554f88382a5ae
I20240501 08:13:14.058960 28678 log_verifier.cc:126] Checking tablet 313e1bed969b4a4ba124116ba5aed741
I20240501 08:13:14.095666 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 313e1bed969b4a4ba124116ba5aed741
I20240501 08:13:14.095782 28678 log_verifier.cc:126] Checking tablet 36c9ad94ddc54bc6ab5021a0d297a774
I20240501 08:13:14.164323 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 36c9ad94ddc54bc6ab5021a0d297a774
I20240501 08:13:14.164413 28678 log_verifier.cc:126] Checking tablet 460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:13:14.200752 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 460948db7dfd4d5589d478ddd5c6c10c
I20240501 08:13:14.200853 28678 log_verifier.cc:126] Checking tablet 4adf393d330f45ba95fb50c850f20348
I20240501 08:13:14.237663 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 4adf393d330f45ba95fb50c850f20348
I20240501 08:13:14.237785 28678 log_verifier.cc:126] Checking tablet 4cb5c6a514354708bff63dd5e51c7420
I20240501 08:13:14.277899 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 4cb5c6a514354708bff63dd5e51c7420
I20240501 08:13:14.278024 28678 log_verifier.cc:126] Checking tablet 6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:13:14.317341 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 6da0f7473ef248f2ad4f5621f3fbd25e
I20240501 08:13:14.317457 28678 log_verifier.cc:126] Checking tablet 6e2593a1a02d4f61972ba63e7dbc628c
I20240501 08:13:14.353940 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 6e2593a1a02d4f61972ba63e7dbc628c
I20240501 08:13:14.354053 28678 log_verifier.cc:126] Checking tablet 7667242ae1c548ba982e74e190f2af27
I20240501 08:13:14.422415 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 7667242ae1c548ba982e74e190f2af27
I20240501 08:13:14.422508 28678 log_verifier.cc:126] Checking tablet 7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:13:14.459688 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 7fde726fbe8d48f685a4580ff6ae4033
I20240501 08:13:14.459807 28678 log_verifier.cc:126] Checking tablet 885de7b67f474f7bb54badfae193906d
I20240501 08:13:14.497546 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 885de7b67f474f7bb54badfae193906d
I20240501 08:13:14.497668 28678 log_verifier.cc:126] Checking tablet 901e7e9545d94ee2904a29c5675ccc75
I20240501 08:13:14.532033 28678 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 901e7e9545d94ee2904a29c5675ccc75
I20240501 08:13:14.532140 28678 log_verifier.cc:126] Checking tablet a3fce8868ca74a028492122fc2762001
I20240501 08:13:14.569555 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet a3fce8868ca74a028492122fc2762001
I20240501 08:13:14.569681 28678 log_verifier.cc:126] Checking tablet a68f4cc374ad4656b1de788033ece98c
I20240501 08:13:14.616297 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet a68f4cc374ad4656b1de788033ece98c
I20240501 08:13:14.616420 28678 log_verifier.cc:126] Checking tablet babe0d774b24482dab935eccb99f31c1
I20240501 08:13:14.690300 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet babe0d774b24482dab935eccb99f31c1
I20240501 08:13:14.690405 28678 log_verifier.cc:126] Checking tablet bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:13:14.734531 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet bdad8a9a16414c5698f9fafb7272bdb1
I20240501 08:13:14.734658 28678 log_verifier.cc:126] Checking tablet c927478b05c14f47b736faf52b4db9aa
I20240501 08:13:14.775362 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet c927478b05c14f47b736faf52b4db9aa
I20240501 08:13:14.775461 28678 log_verifier.cc:126] Checking tablet d110b6fb2855454892ab5359ec3210b0
I20240501 08:13:14.810326 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet d110b6fb2855454892ab5359ec3210b0
I20240501 08:13:14.810432 28678 log_verifier.cc:126] Checking tablet d3db152c64e74ee3a788769dba03b7a6
I20240501 08:13:14.844990 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet d3db152c64e74ee3a788769dba03b7a6
I20240501 08:13:14.845088 28678 log_verifier.cc:126] Checking tablet d56791555f6644a3bcee602299a85761
I20240501 08:13:14.882383 28678 log_verifier.cc:177] Verified matching terms for 10 ops in tablet d56791555f6644a3bcee602299a85761
I20240501 08:13:14.882519 28678 log_verifier.cc:126] Checking tablet d749fd4842294f8fba36f287b7072c9d
I20240501 08:13:14.952730 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet d749fd4842294f8fba36f287b7072c9d
I20240501 08:13:14.952828 28678 log_verifier.cc:126] Checking tablet e21419bf69e746baa7a96fbf5eee384d
I20240501 08:13:14.989624 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet e21419bf69e746baa7a96fbf5eee384d
I20240501 08:13:14.989719 28678 log_verifier.cc:126] Checking tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:13:15.026795 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet e62d72ab5f9b460aaa7fe7bb7ecd7b08
I20240501 08:13:15.026904 28678 log_verifier.cc:126] Checking tablet ea2ecf61668b488c96b903993ab07b16
I20240501 08:13:15.063263 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet ea2ecf61668b488c96b903993ab07b16
I20240501 08:13:15.063362 28678 log_verifier.cc:126] Checking tablet f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:13:15.101826 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet f0c4a54ade9c4e96bb575bfbfa5d0206
I20240501 08:13:15.101931 28678 log_verifier.cc:126] Checking tablet f75a6080c937423f8ffb9f75b96da670
I20240501 08:13:15.139991 28678 log_verifier.cc:177] Verified matching terms for 9 ops in tablet f75a6080c937423f8ffb9f75b96da670
I20240501 08:13:15.140122 28678 log_verifier.cc:126] Checking tablet fdeb6b683c7f4097ba3054ab8d4fb203
I20240501 08:13:15.204324 28678 log_verifier.cc:177] Verified matching terms for 6 ops in tablet fdeb6b683c7f4097ba3054ab8d4fb203
I20240501 08:13:15.205178 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3068
I20240501 08:13:15.216926 3189 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:15.413297 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3068
W20240501 08:13:15.443418 3344 connection.cc:535] server connection from 127.28.1.129:50545 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:13:15.443701 6283 connection.cc:535] server connection from 127.28.1.129:45667 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:13:15.444114 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3199
I20240501 08:13:15.449582 3320 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:13:15.488413 5014 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
W20240501 08:13:15.489854 5014 consensus_peers.cc:485] T babe0d774b24482dab935eccb99f31c1 P f0af5c767c1b496cb1163870b60b066a -> Peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Couldn't send request to peer 466a9d70da1b4b1e8579e40447ce0c25. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:13:15.512316 5146 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
W20240501 08:13:15.513814 5146 consensus_peers.cc:485] T 4cb5c6a514354708bff63dd5e51c7420 P 8151eaf288354d60b73b3c090dd271fd -> Peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Couldn't send request to peer 466a9d70da1b4b1e8579e40447ce0c25. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:13:15.522533 3476 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
W20240501 08:13:15.523883 3476 consensus_peers.cc:485] T 0612f1f8e4ca4a29971151bf52571a33 P 1771142e14884570b63c687e982902aa -> Peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Couldn't send request to peer 466a9d70da1b4b1e8579e40447ce0c25. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:15.675611 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3199
W20240501 08:13:15.678386 3476 consensus_peers.cc:485] T 901e7e9545d94ee2904a29c5675ccc75 P 1771142e14884570b63c687e982902aa -> Peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Couldn't send request to peer 466a9d70da1b4b1e8579e40447ce0c25. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:13:15.689370 5277 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
W20240501 08:13:15.690840 5277 consensus_peers.cc:485] T 885de7b67f474f7bb54badfae193906d P 3e6a44464d7748fa8c784ef795b896fb -> Peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Couldn't send request to peer 466a9d70da1b4b1e8579e40447ce0c25. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:13:15.709630 5147 connection.cc:535] server connection from 127.28.1.130:41195 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:13:15.709751 3474 connection.cc:535] server connection from 127.28.1.130:37631 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:13:15.710142 5410 connection.cc:535] server connection from 127.28.1.130:46267 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:13:15.710268 6280 connection.cc:535] server connection from 127.28.1.130:43831 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:13:15.710516 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3329
I20240501 08:13:15.716701 3451 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:15.917963 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3329
I20240501 08:13:15.948177 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 3459
I20240501 08:13:15.954190 3581 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:16.173341 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 3459
W20240501 08:13:16.184562 6280 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.131:44953: connect: Connection refused (error 111)
W20240501 08:13:16.185931 6280 consensus_peers.cc:485] T 7fde726fbe8d48f685a4580ff6ae4033 P 53ae6b4d587b4c48a6dcc10a7ba67579 -> Peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): Couldn't send request to peer 254a7288f105497eb60d1709036a71a9. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.131:44953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240501 08:13:16.205335 5013 connection.cc:535] server connection from 127.28.1.132:58015 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:13:16.205863 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 4997
I20240501 08:13:16.211357 5118 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:13:16.314675 6280 consensus_peers.cc:485] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 -> Peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): Couldn't send request to peer 254a7288f105497eb60d1709036a71a9. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.131:44953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:16.353842 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 4997
I20240501 08:13:16.374812 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 5128
I20240501 08:13:16.380872 5251 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:16.449560 9551 raft_consensus.cc:482] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 466a9d70da1b4b1e8579e40447ce0c25)
I20240501 08:13:16.449790 9551 raft_consensus.cc:504] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "254a7288f105497eb60d1709036a71a9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 44953 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } }
I20240501 08:13:16.451326 9551 leader_election.cc:288] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953)
W20240501 08:13:16.452527 5409 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
W20240501 08:13:16.453836 5407 leader_election.cc:334] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 254a7288f105497eb60d1709036a71a9 (127.28.1.131:44953): Network error: Client connection negotiation failed: client connection to 127.28.1.131:44953: connect: Connection refused (error 111)
W20240501 08:13:16.454180 5409 leader_election.cc:334] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
I20240501 08:13:16.454335 5409 leader_election.cc:302] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b1a5f0e8d89547a1ac983c6a5256eca8; no voters: 254a7288f105497eb60d1709036a71a9, 466a9d70da1b4b1e8579e40447ce0c25
I20240501 08:13:16.454650 9551 raft_consensus.cc:2726] T 313e1bed969b4a4ba124116ba5aed741 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:13:16.464308 9551 raft_consensus.cc:482] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e3b81e5e25de4eddbeaad1e5834c0836)
I20240501 08:13:16.464483 9551 raft_consensus.cc:504] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:16.465248 9551 leader_election.cc:288] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365)
W20240501 08:13:16.466459 5410 leader_election.cc:334] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): Network error: Client connection negotiation failed: client connection to 127.28.1.130:40531: connect: Connection refused (error 111)
I20240501 08:13:16.521587 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 5128
I20240501 08:13:16.541311 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 5259
W20240501 08:13:16.542073 9553 negotiation.cc:337] Failed RPC negotiation. Trace:
0501 08:13:16.465463 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.28.1.134:44365 (local address 127.28.1.136:47281)
0501 08:13:16.465620 (+ 157us) negotiation.cc:107] Waiting for socket to connect
0501 08:13:16.465637 (+ 17us) client_negotiation.cc:174] Beginning negotiation
0501 08:13:16.465764 (+ 127us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0501 08:13:16.521569 (+ 55805us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
0501 08:13:16.521576 (+ 7us) client_negotiation.cc:394] Received NEGOTIATE response from server
0501 08:13:16.521614 (+ 38us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
0501 08:13:16.522150 (+ 536us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
0501 08:13:16.522163 (+ 13us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
0501 08:13:16.541142 (+ 18979us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.28.1.134:44365: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":63}
W20240501 08:13:16.543274 5409 leader_election.cc:334] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365): Network error: Client connection negotiation failed: client connection to 127.28.1.134:44365: connect: Connection refused (error 111)
I20240501 08:13:16.543447 5409 leader_election.cc:302] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b1a5f0e8d89547a1ac983c6a5256eca8; no voters: 8151eaf288354d60b73b3c090dd271fd, e3b81e5e25de4eddbeaad1e5834c0836
I20240501 08:13:16.543747 9551 raft_consensus.cc:2726] T f75a6080c937423f8ffb9f75b96da670 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:13:16.547505 5382 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:16.561106 9226 raft_consensus.cc:482] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 466a9d70da1b4b1e8579e40447ce0c25)
I20240501 08:13:16.561343 9226 raft_consensus.cc:504] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:16.562150 9226 leader_election.cc:288] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
W20240501 08:13:16.563455 6283 leader_election.cc:334] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
I20240501 08:13:16.588213 9551 raft_consensus.cc:482] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 466a9d70da1b4b1e8579e40447ce0c25)
I20240501 08:13:16.588413 9551 raft_consensus.cc:504] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:16.589154 9551 leader_election.cc:288] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945), 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431)
W20240501 08:13:16.590416 5409 leader_election.cc:334] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 466a9d70da1b4b1e8579e40447ce0c25 (127.28.1.129:42945): Network error: Client connection negotiation failed: client connection to 127.28.1.129:42945: connect: Connection refused (error 111)
I20240501 08:13:16.596185 6350 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a3fce8868ca74a028492122fc2762001" candidate_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" is_pre_election: true
I20240501 08:13:16.596967 5410 leader_election.cc:302] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b1a5f0e8d89547a1ac983c6a5256eca8; no voters: 466a9d70da1b4b1e8579e40447ce0c25, 53ae6b4d587b4c48a6dcc10a7ba67579
I20240501 08:13:16.597249 9551 raft_consensus.cc:2726] T a3fce8868ca74a028492122fc2762001 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:13:16.690009 9544 raft_consensus.cc:482] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 466a9d70da1b4b1e8579e40447ce0c25)
I20240501 08:13:16.690164 5340 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e62d72ab5f9b460aaa7fe7bb7ecd7b08" candidate_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "3e6a44464d7748fa8c784ef795b896fb" is_pre_election: true
I20240501 08:13:16.690266 9544 raft_consensus.cc:504] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:16.690819 5340 raft_consensus.cc:2454] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 3e6a44464d7748fa8c784ef795b896fb [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53ae6b4d587b4c48a6dcc10a7ba67579 in term 1.
I20240501 08:13:16.690831 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 5259
I20240501 08:13:16.711583 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 5390
W20240501 08:13:16.712052 6281 leader_election.cc:334] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375): Network error: Client connection negotiation failed: client connection to 127.28.1.135:42375: connect: Connection refused (error 111)
I20240501 08:13:16.712211 6281 leader_election.cc:302] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53ae6b4d587b4c48a6dcc10a7ba67579; no voters: 3e6a44464d7748fa8c784ef795b896fb, 466a9d70da1b4b1e8579e40447ce0c25
I20240501 08:13:16.712579 9560 raft_consensus.cc:2726] T e62d72ab5f9b460aaa7fe7bb7ecd7b08 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:13:16.717715 5514 generic_service.cc:196] Checking for leaks (request via RPC)
W20240501 08:13:16.833385 6280 consensus_peers.cc:485] T d56791555f6644a3bcee602299a85761 P 53ae6b4d587b4c48a6dcc10a7ba67579 -> Peer f0af5c767c1b496cb1163870b60b066a (127.28.1.133:43351): Couldn't send request to peer f0af5c767c1b496cb1163870b60b066a. Status: Network error: Client connection negotiation failed: client connection to 127.28.1.133:43351: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:16.859951 9551 raft_consensus.cc:482] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e3b81e5e25de4eddbeaad1e5834c0836)
I20240501 08:13:16.860280 9563 raft_consensus.cc:482] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader e3b81e5e25de4eddbeaad1e5834c0836)
I20240501 08:13:16.860229 9551 raft_consensus.cc:504] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:16.860466 9563 raft_consensus.cc:504] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e3b81e5e25de4eddbeaad1e5834c0836" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 40531 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "8151eaf288354d60b73b3c090dd271fd" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 44365 } attrs { promote: false } }
I20240501 08:13:16.861091 9564 raft_consensus.cc:482] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1771142e14884570b63c687e982902aa)
I20240501 08:13:16.861171 9551 leader_election.cc:288] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 53ae6b4d587b4c48a6dcc10a7ba67579 (127.28.1.137:42431)
I20240501 08:13:16.861251 9564 raft_consensus.cc:504] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1771142e14884570b63c687e982902aa" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43555 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "3e6a44464d7748fa8c784ef795b896fb" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 42375 } attrs { promote: false } }
I20240501 08:13:16.862002 9563 leader_election.cc:288] T c927478b05c14f47b736faf52b4db9aa P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531), 8151eaf288354d60b73b3c090dd271fd (127.28.1.134:44365)
I20240501 08:13:16.861968 6350 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d749fd4842294f8fba36f287b7072c9d" candidate_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" is_pre_election: true
W20240501 08:13:16.862545 5410 leader_election.cc:334] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e3b81e5e25de4eddbeaad1e5834c0836 (127.28.1.130:40531): Network error: Client connection negotiation failed: client connection to 127.28.1.130:40531: connect: Connection refused (error 111)
I20240501 08:13:16.862661 9564 leader_election.cc:288] T 0434084d13f740368d1daae53dee9fe2 P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1771142e14884570b63c687e982902aa (127.28.1.132:43555), 3e6a44464d7748fa8c784ef795b896fb (127.28.1.135:42375)
I20240501 08:13:16.862717 5410 leader_election.cc:302] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b1a5f0e8d89547a1ac983c6a5256eca8; no voters: 53ae6b4d587b4c48a6dcc10a7ba67579, e3b81e5e25de4eddbeaad1e5834c0836
I20240501 08:13:16.862996 9564 raft_consensus.cc:2726] T d749fd4842294f8fba36f287b7072c9d P b1a5f0e8d89547a1ac983c6a5256eca8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240501 08:13:16.863236 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 5390
I20240501 08:13:16.885053 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 6152
I20240501 08:13:16.890765 6402 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:17.016304 9560 raft_consensus.cc:482] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 466a9d70da1b4b1e8579e40447ce0c25)
I20240501 08:13:17.016578 9560 raft_consensus.cc:504] T a3fce8868ca74a028492122fc2762001 P 53ae6b4d587b4c48a6dcc10a7ba67579 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "466a9d70da1b4b1e8579e40447ce0c25" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 42945 } } peers { permanent_uuid: "b1a5f0e8d89547a1ac983c6a5256eca8" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 42523 } attrs { promote: false } } peers { permanent_uuid: "53ae6b4d587b4c48a6dcc10a7ba67579" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 42431 } attrs { promote: false } }
I20240501 08:13:17.017082 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 6152
I20240501 08:13:17.035188 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 2978
I20240501 08:13:17.036207 3038 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:17.223356 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 2978
2024-05-01T08:13:17Z chronyd exiting
[ OK ] TserverAddedDuringRebalancingTest.TserverStarts/1 (49793 ms)
[----------] 1 test from TserverAddedDuringRebalancingTest (49793 ms total)
[----------] 1 test from RebalancerAndSingleReplicaTablets
[ RUN ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5
I20240501 08:13:17.363361 28678 test_util.cc:268] Using random seed: 252800139
I20240501 08:13:17.365114 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:13:17.365181 28678 ts_itest-base.cc:113] --------------
I20240501 08:13:17.365221 28678 ts_itest-base.cc:114] 4 tablet servers
I20240501 08:13:17.365252 28678 ts_itest-base.cc:115] 1 replicas per TS
I20240501 08:13:17.365298 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:13:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:13:17Z Disabled control of system clock
I20240501 08:13:17.409123 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:46033
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:46033
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false with env {}
W20240501 08:13:17.726754 9574 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:17.727067 9574 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:17.727155 9574 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:17.734555 9574 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:17.734684 9574 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:13:17.734730 9574 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:17.734786 9574 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:13:17.734833 9574 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:13:17.743983 9574 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:46033
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:46033
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:17.745210 9574 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:17.746857 9574 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:17.757133 9582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:17.757539 9579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:17.757979 9580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:17.758678 9574 server_base.cc:1034] running on GCE node
I20240501 08:13:17.759402 9574 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:17.760819 9574 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:17.762017 9574 hybrid_clock.cc:648] HybridClock initialized: now 1714551197761965 us; error 46 us; skew 500 ppm
I20240501 08:13:17.762323 9574 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:17.764575 9574 webserver.cc:460] Webserver started at http://127.28.1.190:41475/ using document root <none> and password file <none>
I20240501 08:13:17.765283 9574 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:17.765379 9574 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:17.765635 9574 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:17.768157 9574 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "12f90df9121740219a05e9a58a61d52c"
format_stamp: "Formatted at 2024-05-01 08:13:17 on dist-test-slave-jj4n"
I20240501 08:13:17.768858 9574 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "12f90df9121740219a05e9a58a61d52c"
format_stamp: "Formatted at 2024-05-01 08:13:17 on dist-test-slave-jj4n"
I20240501 08:13:17.773766 9574 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:13:17.776966 9588 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:17.778287 9574 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:17.778426 9574 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "12f90df9121740219a05e9a58a61d52c"
format_stamp: "Formatted at 2024-05-01 08:13:17 on dist-test-slave-jj4n"
I20240501 08:13:17.778594 9574 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:17.824926 9574 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:17.825757 9574 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:17.825991 9574 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:17.842258 9574 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:46033
I20240501 08:13:17.842284 9639 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:46033 every 8 connection(s)
I20240501 08:13:17.843580 9574 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:13:17.844709 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 9574
I20240501 08:13:17.844909 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:13:17.848076 9640 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:17.858915 9640 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c: Bootstrap starting.
I20240501 08:13:17.861677 9640 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:17.862718 9640 log.cc:824] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:17.865017 9640 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c: No bootstrap required, opened a new log
I20240501 08:13:17.870841 9640 raft_consensus.cc:348] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } }
I20240501 08:13:17.871148 9640 raft_consensus.cc:374] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:17.871223 9640 raft_consensus.cc:729] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12f90df9121740219a05e9a58a61d52c, State: Initialized, Role: FOLLOWER
I20240501 08:13:17.871716 9640 consensus_queue.cc:260] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } }
I20240501 08:13:17.871899 9640 raft_consensus.cc:388] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:17.872035 9640 raft_consensus.cc:482] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:17.872159 9640 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:17.874137 9640 raft_consensus.cc:504] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } }
I20240501 08:13:17.874542 9640 leader_election.cc:302] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12f90df9121740219a05e9a58a61d52c; no voters:
I20240501 08:13:17.875046 9640 leader_election.cc:288] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:17.875172 9645 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:17.875696 9645 raft_consensus.cc:686] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [term 1 LEADER]: Becoming Leader. State: Replica: 12f90df9121740219a05e9a58a61d52c, State: Running, Role: LEADER
I20240501 08:13:17.876228 9645 consensus_queue.cc:237] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } }
I20240501 08:13:17.876828 9640 sys_catalog.cc:564] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:13:17.878821 9647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 12f90df9121740219a05e9a58a61d52c. Latest consensus state: current_term: 1 leader_uuid: "12f90df9121740219a05e9a58a61d52c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } } }
I20240501 08:13:17.879142 9647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:17.879976 9646 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "12f90df9121740219a05e9a58a61d52c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12f90df9121740219a05e9a58a61d52c" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 46033 } } }
I20240501 08:13:17.880498 9646 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:17.880849 9651 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:13:17.885985 9651 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:13:17.894234 9651 catalog_manager.cc:1347] Generated new cluster ID: 0eb301f0978247959330c48bc0b8c522
I20240501 08:13:17.894356 9651 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:13:17.910599 9651 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:13:17.912086 9651 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:13:17.933712 9651 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 12f90df9121740219a05e9a58a61d52c: Generated new TSK 0
I20240501 08:13:17.934541 9651 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:13:17.954582 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:13:18.275951 9664 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:18.276331 9664 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:18.276476 9664 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:18.284030 9664 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:18.284134 9664 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:18.284255 9664 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:13:18.293031 9664 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:18.294211 9664 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:18.295812 9664 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:18.307265 9669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:18.307754 9670 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:18.308140 9672 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:18.308738 9664 server_base.cc:1034] running on GCE node
I20240501 08:13:18.309293 9664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:18.310184 9664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:18.311381 9664 hybrid_clock.cc:648] HybridClock initialized: now 1714551198311332 us; error 36 us; skew 500 ppm
I20240501 08:13:18.311668 9664 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:18.313796 9664 webserver.cc:460] Webserver started at http://127.28.1.129:39193/ using document root <none> and password file <none>
I20240501 08:13:18.314476 9664 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:18.314589 9664 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:18.314886 9664 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:18.317343 9664 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "c5ea0089ed5940089fb30561112e965a"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.318045 9664 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "c5ea0089ed5940089fb30561112e965a"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.322749 9664 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20240501 08:13:18.325847 9678 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.327015 9664 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240501 08:13:18.327157 9664 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "c5ea0089ed5940089fb30561112e965a"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.327330 9664 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:18.358999 9664 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:18.359894 9664 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:18.360169 9664 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:18.361191 9664 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:18.362727 9664 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:18.362819 9664 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.362955 9664 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:18.363018 9664 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.392082 9664 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:36189
I20240501 08:13:18.392151 9790 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:36189 every 8 connection(s)
I20240501 08:13:18.393345 9664 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:13:18.400691 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 9664
I20240501 08:13:18.400898 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:13:18.404192 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:18.405433 9791 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:18.405711 9791 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:18.406478 9791 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:18.408598 9605 ts_manager.cc:194] Registered new tserver with Master: c5ea0089ed5940089fb30561112e965a (127.28.1.129:36189)
I20240501 08:13:18.410492 9605 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:57383
W20240501 08:13:18.722136 9795 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:18.722467 9795 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:18.722606 9795 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:18.729990 9795 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:18.730150 9795 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:18.730319 9795 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:13:18.739322 9795 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:18.740689 9795 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:18.742385 9795 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:18.753371 9800 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:18.753741 9803 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:18.754258 9801 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:18.755060 9795 server_base.cc:1034] running on GCE node
I20240501 08:13:18.756939 9795 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:18.758416 9795 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:18.759630 9795 hybrid_clock.cc:648] HybridClock initialized: now 1714551198759586 us; error 36 us; skew 500 ppm
I20240501 08:13:18.759954 9795 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:18.762208 9795 webserver.cc:460] Webserver started at http://127.28.1.130:35183/ using document root <none> and password file <none>
I20240501 08:13:18.762945 9795 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:18.763062 9795 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:18.763367 9795 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:18.765889 9795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "f22575e178c54dccae785f81ceff1f04"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.766616 9795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "f22575e178c54dccae785f81ceff1f04"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.771605 9795 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240501 08:13:18.774761 9809 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.776135 9795 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:18.776315 9795 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "f22575e178c54dccae785f81ceff1f04"
format_stamp: "Formatted at 2024-05-01 08:13:18 on dist-test-slave-jj4n"
I20240501 08:13:18.776496 9795 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:18.801620 9795 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:18.802470 9795 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:18.802742 9795 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:18.803694 9795 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:18.805155 9795 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:18.805254 9795 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.805359 9795 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:18.805428 9795 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:18.834189 9795 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:38835
I20240501 08:13:18.834234 9921 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:38835 every 8 connection(s)
I20240501 08:13:18.835427 9795 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:13:18.841629 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 9795
I20240501 08:13:18.841775 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:13:18.844941 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:18.848587 9922 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:18.848910 9922 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:18.849821 9922 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:18.851372 9605 ts_manager.cc:194] Registered new tserver with Master: f22575e178c54dccae785f81ceff1f04 (127.28.1.130:38835)
I20240501 08:13:18.852427 9605 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:43693
W20240501 08:13:19.168937 9925 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:19.169230 9925 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:19.169343 9925 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:19.176610 9925 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:19.176760 9925 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:19.176904 9925 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:13:19.186098 9925 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:19.187371 9925 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:19.189026 9925 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:19.200311 9933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:19.200735 9930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:19.201249 9931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:19.201879 9925 server_base.cc:1034] running on GCE node
I20240501 08:13:19.202467 9925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:19.203372 9925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:19.204560 9925 hybrid_clock.cc:648] HybridClock initialized: now 1714551199204514 us; error 41 us; skew 500 ppm
I20240501 08:13:19.204855 9925 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:19.206987 9925 webserver.cc:460] Webserver started at http://127.28.1.131:42383/ using document root <none> and password file <none>
I20240501 08:13:19.207697 9925 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:19.207813 9925 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:19.208144 9925 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:19.210669 9925 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "199ef6a56e714a3894a5acbf81644839"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.211426 9925 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "199ef6a56e714a3894a5acbf81644839"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.216488 9925 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
I20240501 08:13:19.219868 9939 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.221246 9925 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:19.221421 9925 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "199ef6a56e714a3894a5acbf81644839"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.221618 9925 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:19.238729 9925 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:19.239573 9925 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:19.239836 9925 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:19.240845 9925 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:19.242305 9925 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:19.242396 9925 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.242503 9925 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:19.242569 9925 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.273080 9925 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:37067
I20240501 08:13:19.273129 10051 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:37067 every 8 connection(s)
I20240501 08:13:19.274375 9925 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:13:19.283181 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 9925
I20240501 08:13:19.283370 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:13:19.286801 10052 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:19.286809 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:19.287097 10052 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:19.287863 10052 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:19.289314 9605 ts_manager.cc:194] Registered new tserver with Master: 199ef6a56e714a3894a5acbf81644839 (127.28.1.131:37067)
I20240501 08:13:19.290066 9605 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:42107
I20240501 08:13:19.413332 9791 heartbeater.cc:503] Master 127.28.1.190:46033 was elected leader, sending a full tablet report...
W20240501 08:13:19.619741 10055 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:19.620151 10055 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:19.620293 10055 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:19.627907 10055 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:19.628069 10055 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:19.628206 10055 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:13:19.637284 10055 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:19.638577 10055 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:19.640285 10055 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:19.651907 10060 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:19.652235 10063 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:19.652750 10061 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:19.653329 10055 server_base.cc:1034] running on GCE node
I20240501 08:13:19.653945 10055 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:19.654903 10055 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:19.656096 10055 hybrid_clock.cc:648] HybridClock initialized: now 1714551199656045 us; error 45 us; skew 500 ppm
I20240501 08:13:19.656419 10055 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:19.658576 10055 webserver.cc:460] Webserver started at http://127.28.1.132:45309/ using document root <none> and password file <none>
I20240501 08:13:19.659293 10055 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:19.659417 10055 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:19.659711 10055 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:19.664920 10055 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "efa14afe2d28444898677bc8c15b4dc4"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.665664 10055 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "efa14afe2d28444898677bc8c15b4dc4"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.670676 10055 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:13:19.674015 10069 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.675289 10055 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:19.675448 10055 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "efa14afe2d28444898677bc8c15b4dc4"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:19.675627 10055 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:19.691182 10055 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:19.691998 10055 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:19.692245 10055 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:19.693143 10055 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:19.694557 10055 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:19.694650 10055 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.694753 10055 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:19.694819 10055 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:19.723817 10055 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:40459
I20240501 08:13:19.723863 10181 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:40459 every 8 connection(s)
I20240501 08:13:19.725173 10055 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:13:19.734124 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10055
I20240501 08:13:19.734313 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:13:19.737285 10182 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:19.737583 10182 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:19.738402 10182 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:19.739617 9605 ts_manager.cc:194] Registered new tserver with Master: efa14afe2d28444898677bc8c15b4dc4 (127.28.1.132:40459)
I20240501 08:13:19.740440 9605 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:36901
I20240501 08:13:19.749994 28678 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240501 08:13:19.764504 9605 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46054:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240501 08:13:19.790021 9987 tablet_service.cc:1467] Processing CreateTablet for tablet dc2cfd97babd4e1d8a23233e05d2b9c4 (DEFAULT_TABLE table=TestTable [id=62a077b08c3142aeb000d78976386443]), partition=RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:19.791244 9987 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dc2cfd97babd4e1d8a23233e05d2b9c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:19.800904 10196 tablet_bootstrap.cc:492] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Bootstrap starting.
I20240501 08:13:19.803861 10196 tablet_bootstrap.cc:654] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:19.804826 10196 log.cc:824] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:19.807099 10196 tablet_bootstrap.cc:492] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: No bootstrap required, opened a new log
I20240501 08:13:19.807353 10196 ts_tablet_manager.cc:1367] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:13:19.813396 10196 raft_consensus.cc:348] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:19.813719 10196 raft_consensus.cc:374] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:19.813807 10196 raft_consensus.cc:729] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 199ef6a56e714a3894a5acbf81644839, State: Initialized, Role: FOLLOWER
I20240501 08:13:19.814383 10196 consensus_queue.cc:260] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:19.814579 10196 raft_consensus.cc:388] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:19.814692 10196 raft_consensus.cc:482] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:19.814829 10196 raft_consensus.cc:3037] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:19.816794 10196 raft_consensus.cc:504] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:19.817234 10196 leader_election.cc:302] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 199ef6a56e714a3894a5acbf81644839; no voters:
I20240501 08:13:19.817777 10196 leader_election.cc:288] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:19.817920 10198 raft_consensus.cc:2781] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:19.818926 10052 heartbeater.cc:503] Master 127.28.1.190:46033 was elected leader, sending a full tablet report...
I20240501 08:13:19.819494 10196 ts_tablet_manager.cc:1398] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20240501 08:13:19.819669 10198 raft_consensus.cc:686] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 LEADER]: Becoming Leader. State: Replica: 199ef6a56e714a3894a5acbf81644839, State: Running, Role: LEADER
I20240501 08:13:19.820123 10198 consensus_queue.cc:237] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:19.825148 9605 catalog_manager.cc:5484] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 reported cstate change: term changed from 0 to 1, leader changed from <none> to 199ef6a56e714a3894a5acbf81644839 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "199ef6a56e714a3894a5acbf81644839" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:19.834497 28678 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240501 08:13:19.836344 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver c5ea0089ed5940089fb30561112e965a to finish bootstrapping
I20240501 08:13:19.845676 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver f22575e178c54dccae785f81ceff1f04 to finish bootstrapping
I20240501 08:13:19.855088 9922 heartbeater.cc:503] Master 127.28.1.190:46033 was elected leader, sending a full tablet report...
I20240501 08:13:19.855548 28678 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 199ef6a56e714a3894a5acbf81644839 to finish bootstrapping
I20240501 08:13:19.862648 28678 ts_itest-base.cc:233] Waiting for 0 tablets on tserver efa14afe2d28444898677bc8c15b4dc4 to finish bootstrapping
I20240501 08:13:19.870887 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 9925
I20240501 08:13:19.871750 10047 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:19.954582 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 9925
I20240501 08:13:19.967859 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10055
I20240501 08:13:19.968943 10177 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:20.046756 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10055
I20240501 08:13:23.811349 9605 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46054:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:13:23.830858 9725 tablet_service.cc:1467] Processing CreateTablet for tablet 3684ee20d5a2468eb25c78adec1a61bc (DEFAULT_TABLE table=rebalance_test_table_0 [id=85e8145c01224d89bfa94f8c081bde47]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:23.830804 9726 tablet_service.cc:1467] Processing CreateTablet for tablet 957112d49396418cb624aa2dd72562ce (DEFAULT_TABLE table=rebalance_test_table_0 [id=85e8145c01224d89bfa94f8c081bde47]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:23.831338 9857 tablet_service.cc:1467] Processing CreateTablet for tablet 6dbd39eba86a4c09839a9065220d6342 (DEFAULT_TABLE table=rebalance_test_table_0 [id=85e8145c01224d89bfa94f8c081bde47]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:23.832353 9725 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3684ee20d5a2468eb25c78adec1a61bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:23.832643 9857 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6dbd39eba86a4c09839a9065220d6342. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:23.832752 9726 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 957112d49396418cb624aa2dd72562ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:23.848073 10218 tablet_bootstrap.cc:492] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:23.848526 10217 tablet_bootstrap.cc:492] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:23.851011 10218 tablet_bootstrap.cc:654] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:23.851647 10217 tablet_bootstrap.cc:654] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:23.852114 10218 log.cc:824] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:23.852839 10217 log.cc:824] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:23.855185 10217 tablet_bootstrap.cc:492] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:23.855322 10218 tablet_bootstrap.cc:492] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:23.855437 10217 ts_tablet_manager.cc:1367] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.002s
I20240501 08:13:23.855566 10218 ts_tablet_manager.cc:1367] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20240501 08:13:23.861258 10217 raft_consensus.cc:348] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:23.861578 10217 raft_consensus.cc:374] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:23.861685 10217 raft_consensus.cc:729] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:23.862357 10217 consensus_queue.cc:260] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:23.862569 10217 raft_consensus.cc:388] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:23.862705 10217 raft_consensus.cc:482] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:23.862519 10218 raft_consensus.cc:348] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.862865 10218 raft_consensus.cc:374] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:23.862879 10217 raft_consensus.cc:3037] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:23.863008 10218 raft_consensus.cc:729] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:23.863658 10218 consensus_queue.cc:260] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.863938 10218 raft_consensus.cc:388] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:23.864117 10218 raft_consensus.cc:482] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:23.864277 10218 raft_consensus.cc:3037] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:23.865188 10217 raft_consensus.cc:504] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:23.865631 10217 leader_election.cc:302] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:23.866168 10217 leader_election.cc:288] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:23.866344 10221 raft_consensus.cc:2781] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:23.866340 10218 raft_consensus.cc:504] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.866931 10218 leader_election.cc:302] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:23.867689 10217 ts_tablet_manager.cc:1398] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.012s user 0.008s sys 0.005s
I20240501 08:13:23.868263 10218 leader_election.cc:288] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:23.868337 10224 raft_consensus.cc:2781] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:23.869864 10218 ts_tablet_manager.cc:1398] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.014s user 0.015s sys 0.000s
I20240501 08:13:23.870275 10218 tablet_bootstrap.cc:492] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:23.872068 10218 tablet_bootstrap.cc:654] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:23.873275 10221 raft_consensus.cc:686] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:23.873766 10221 consensus_queue.cc:237] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:23.874284 10218 tablet_bootstrap.cc:492] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:23.874428 10218 ts_tablet_manager.cc:1367] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240501 08:13:23.875380 10224 raft_consensus.cc:686] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:23.875383 10218 raft_consensus.cc:348] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.875696 10218 raft_consensus.cc:374] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:23.875792 10218 raft_consensus.cc:729] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:23.876055 10218 consensus_queue.cc:260] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.875955 10224 consensus_queue.cc:237] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.876221 10218 raft_consensus.cc:388] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:23.876300 10218 raft_consensus.cc:482] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:23.876416 10218 raft_consensus.cc:3037] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:23.878440 9605 catalog_manager.cc:5484] T 6dbd39eba86a4c09839a9065220d6342 P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:23.878913 10218 raft_consensus.cc:504] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.879194 10218 leader_election.cc:302] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:23.879384 10218 leader_election.cc:288] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:23.879606 10225 raft_consensus.cc:2781] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:23.882256 10225 raft_consensus.cc:686] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:23.882558 10225 consensus_queue.cc:237] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:23.882871 9604 catalog_manager.cc:5484] T 957112d49396418cb624aa2dd72562ce P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:23.886724 10218 ts_tablet_manager.cc:1398] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.012s user 0.005s sys 0.001s
I20240501 08:13:23.893437 9604 catalog_manager.cc:5484] T 3684ee20d5a2468eb25c78adec1a61bc P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:24.393548 9604 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46054:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:13:24.401926 9857 tablet_service.cc:1467] Processing CreateTablet for tablet 4ab60424862b4f3484d7bae7e655beb4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=b889f10bc18e4e4e8e98063670246bcc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:24.402165 9856 tablet_service.cc:1467] Processing CreateTablet for tablet a876452417414e9cbb0dd80fc75a8c8b (DEFAULT_TABLE table=rebalance_test_table_1 [id=b889f10bc18e4e4e8e98063670246bcc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:24.402390 9857 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ab60424862b4f3484d7bae7e655beb4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:24.402374 9726 tablet_service.cc:1467] Processing CreateTablet for tablet b056930091f64e908cb0a8285e478d67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=b889f10bc18e4e4e8e98063670246bcc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:24.402611 9856 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a876452417414e9cbb0dd80fc75a8c8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:24.402727 9726 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b056930091f64e908cb0a8285e478d67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:24.407665 10258 tablet_bootstrap.cc:492] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:24.409384 10258 tablet_bootstrap.cc:654] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:24.410706 10259 tablet_bootstrap.cc:492] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:24.411432 10258 tablet_bootstrap.cc:492] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:24.411561 10258 ts_tablet_manager.cc:1367] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240501 08:13:24.412258 10259 tablet_bootstrap.cc:654] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:24.412355 10258 raft_consensus.cc:348] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.412500 10258 raft_consensus.cc:374] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:24.412585 10258 raft_consensus.cc:729] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:24.412789 10258 consensus_queue.cc:260] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.412933 10258 raft_consensus.cc:388] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:24.413014 10258 raft_consensus.cc:482] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:24.413112 10258 raft_consensus.cc:3037] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:24.414095 10259 tablet_bootstrap.cc:492] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:24.414224 10259 ts_tablet_manager.cc:1367] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:13:24.414883 10259 raft_consensus.cc:348] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:24.414933 10258 raft_consensus.cc:504] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.415048 10259 raft_consensus.cc:374] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:24.415136 10259 raft_consensus.cc:729] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:24.415154 10258 leader_election.cc:302] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:24.415359 10259 consensus_queue.cc:260] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:24.415581 10259 raft_consensus.cc:388] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:24.415668 10259 raft_consensus.cc:482] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:24.415778 10259 raft_consensus.cc:3037] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:24.418547 10259 raft_consensus.cc:504] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:24.418780 10259 leader_election.cc:302] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:24.418956 10259 leader_election.cc:288] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:24.419061 10225 raft_consensus.cc:2781] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:24.421293 10258 leader_election.cc:288] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:24.421445 10262 raft_consensus.cc:2781] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:24.421941 10225 raft_consensus.cc:686] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:24.422209 10225 consensus_queue.cc:237] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:24.425140 10262 raft_consensus.cc:686] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:24.425464 10262 consensus_queue.cc:237] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.425694 10259 ts_tablet_manager.cc:1398] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.011s user 0.004s sys 0.007s
I20240501 08:13:24.428236 10258 ts_tablet_manager.cc:1398] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.017s user 0.000s sys 0.016s
I20240501 08:13:24.428494 10258 tablet_bootstrap.cc:492] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:24.428825 9605 catalog_manager.cc:5484] T b056930091f64e908cb0a8285e478d67 P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:24.428825 9604 catalog_manager.cc:5484] T 4ab60424862b4f3484d7bae7e655beb4 P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:24.429752 10258 tablet_bootstrap.cc:654] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:24.431824 10258 tablet_bootstrap.cc:492] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:24.431999 10258 ts_tablet_manager.cc:1367] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:13:24.432883 10258 raft_consensus.cc:348] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.433089 10258 raft_consensus.cc:374] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:24.433158 10258 raft_consensus.cc:729] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:24.433897 10258 consensus_queue.cc:260] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.434078 10258 raft_consensus.cc:388] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:24.434168 10258 raft_consensus.cc:482] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:24.434258 10258 raft_consensus.cc:3037] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:24.436947 10258 raft_consensus.cc:504] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.437170 10258 leader_election.cc:302] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:24.437369 10258 leader_election.cc:288] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:24.437613 10262 raft_consensus.cc:2781] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:24.437785 10262 raft_consensus.cc:686] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:24.437816 10258 ts_tablet_manager.cc:1398] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.006s user 0.002s sys 0.001s
I20240501 08:13:24.437999 10262 consensus_queue.cc:237] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:24.442404 9603 catalog_manager.cc:5484] T a876452417414e9cbb0dd80fc75a8c8b P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:25.776134 9603 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46054:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:13:25.784679 9726 tablet_service.cc:1467] Processing CreateTablet for tablet c2a09ad35b82460587ccbcbca618ca73 (DEFAULT_TABLE table=rebalance_test_table_2 [id=51da62913bff4ae38e9428f21b000bd5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:25.784750 9856 tablet_service.cc:1467] Processing CreateTablet for tablet aa14875486264c67bc7e2cca9e6f1af7 (DEFAULT_TABLE table=rebalance_test_table_2 [id=51da62913bff4ae38e9428f21b000bd5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:25.784962 9725 tablet_service.cc:1467] Processing CreateTablet for tablet c76dd2c00c21491387b5ae952534325f (DEFAULT_TABLE table=rebalance_test_table_2 [id=51da62913bff4ae38e9428f21b000bd5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:25.785144 9726 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c2a09ad35b82460587ccbcbca618ca73. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:25.785178 9856 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa14875486264c67bc7e2cca9e6f1af7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:25.785427 9725 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c76dd2c00c21491387b5ae952534325f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:25.791497 10300 tablet_bootstrap.cc:492] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:25.791883 10299 tablet_bootstrap.cc:492] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:25.793143 10300 tablet_bootstrap.cc:654] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:25.793269 10299 tablet_bootstrap.cc:654] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:25.795012 10300 tablet_bootstrap.cc:492] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:25.795017 10299 tablet_bootstrap.cc:492] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:25.795142 10300 ts_tablet_manager.cc:1367] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:13:25.795153 10299 ts_tablet_manager.cc:1367] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:25.795790 10299 raft_consensus.cc:348] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.795805 10300 raft_consensus.cc:348] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:25.795928 10300 raft_consensus.cc:374] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:25.795929 10299 raft_consensus.cc:374] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:25.796027 10300 raft_consensus.cc:729] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:25.796046 10299 raft_consensus.cc:729] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:25.796229 10300 consensus_queue.cc:260] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:25.796238 10299 consensus_queue.cc:260] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.796378 10300 raft_consensus.cc:388] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:25.796393 10299 raft_consensus.cc:388] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:25.796458 10300 raft_consensus.cc:482] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:25.796502 10299 raft_consensus.cc:482] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:25.796561 10300 raft_consensus.cc:3037] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:25.796612 10299 raft_consensus.cc:3037] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:25.798259 10300 raft_consensus.cc:504] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:25.798270 10299 raft_consensus.cc:504] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.798476 10300 leader_election.cc:302] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:25.798481 10299 leader_election.cc:302] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:25.798903 10299 leader_election.cc:288] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:25.799171 10300 leader_election.cc:288] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:25.799278 10303 raft_consensus.cc:2781] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:25.799643 10304 raft_consensus.cc:2781] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:25.800026 10303 raft_consensus.cc:686] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:25.800563 10304 raft_consensus.cc:686] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:25.800834 10304 consensus_queue.cc:237] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.801776 10299 ts_tablet_manager.cc:1398] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.007s user 0.005s sys 0.000s
I20240501 08:13:25.802073 10299 tablet_bootstrap.cc:492] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:25.803726 10299 tablet_bootstrap.cc:654] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:25.804172 10303 consensus_queue.cc:237] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:25.806069 9603 catalog_manager.cc:5484] T c2a09ad35b82460587ccbcbca618ca73 P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:25.810904 10300 ts_tablet_manager.cc:1398] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.016s user 0.005s sys 0.008s
I20240501 08:13:25.810804 9604 catalog_manager.cc:5484] T aa14875486264c67bc7e2cca9e6f1af7 P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:25.812261 10299 tablet_bootstrap.cc:492] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:25.812454 10299 ts_tablet_manager.cc:1367] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.004s
I20240501 08:13:25.813040 10299 raft_consensus.cc:348] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.813205 10299 raft_consensus.cc:374] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:25.813280 10299 raft_consensus.cc:729] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:25.813499 10299 consensus_queue.cc:260] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.813634 10299 raft_consensus.cc:388] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:25.814098 10299 raft_consensus.cc:482] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:25.814257 10299 raft_consensus.cc:3037] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:25.816629 10299 raft_consensus.cc:504] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.816816 10299 leader_election.cc:302] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:25.817006 10299 leader_election.cc:288] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:25.817090 10308 raft_consensus.cc:2781] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:25.817247 10308 raft_consensus.cc:686] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:25.817420 10299 ts_tablet_manager.cc:1398] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:13:25.817512 10308 consensus_queue.cc:237] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:25.820950 9603 catalog_manager.cc:5484] T c76dd2c00c21491387b5ae952534325f P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:26.304983 9603 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46054:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240501 08:13:26.312058 9857 tablet_service.cc:1467] Processing CreateTablet for tablet 997c9d486efb4dd08680b42268c5aef4 (DEFAULT_TABLE table=rebalance_test_table_3 [id=cad44d6b7a1f4bd68a734f6c3d654bea]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:26.312067 9856 tablet_service.cc:1467] Processing CreateTablet for tablet dd5729108dc84d80a5dfe37a57298edc (DEFAULT_TABLE table=rebalance_test_table_3 [id=cad44d6b7a1f4bd68a734f6c3d654bea]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:26.312610 9857 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 997c9d486efb4dd08680b42268c5aef4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:26.312839 9856 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd5729108dc84d80a5dfe37a57298edc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:26.312806 9726 tablet_service.cc:1467] Processing CreateTablet for tablet ea0c2646c6b045fa9b4c75c04526f51f (DEFAULT_TABLE table=rebalance_test_table_3 [id=cad44d6b7a1f4bd68a734f6c3d654bea]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:26.313181 9726 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ea0c2646c6b045fa9b4c75c04526f51f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:26.319370 10337 tablet_bootstrap.cc:492] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a: Bootstrap starting.
I20240501 08:13:26.319952 10336 tablet_bootstrap.cc:492] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:26.320804 10337 tablet_bootstrap.cc:654] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:26.321103 10336 tablet_bootstrap.cc:654] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:26.322582 10337 tablet_bootstrap.cc:492] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a: No bootstrap required, opened a new log
I20240501 08:13:26.322721 10337 ts_tablet_manager.cc:1367] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:26.322943 10336 tablet_bootstrap.cc:492] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:26.323074 10336 ts_tablet_manager.cc:1367] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:26.323428 10337 raft_consensus.cc:348] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:26.323612 10337 raft_consensus.cc:374] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:26.323714 10337 raft_consensus.cc:729] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Initialized, Role: FOLLOWER
I20240501 08:13:26.323730 10336 raft_consensus.cc:348] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.323879 10336 raft_consensus.cc:374] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:26.323997 10336 raft_consensus.cc:729] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:26.323940 10337 consensus_queue.cc:260] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:26.324143 10337 raft_consensus.cc:388] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:26.324218 10337 raft_consensus.cc:482] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:26.324229 10336 consensus_queue.cc:260] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.324321 10337 raft_consensus.cc:3037] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:26.324399 10336 raft_consensus.cc:388] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:26.324472 10336 raft_consensus.cc:482] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:26.324575 10336 raft_consensus.cc:3037] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:26.326854 10337 raft_consensus.cc:504] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:26.326948 10336 raft_consensus.cc:504] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.327095 10337 leader_election.cc:302] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: c5ea0089ed5940089fb30561112e965a; no voters:
I20240501 08:13:26.327158 10336 leader_election.cc:302] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:26.327286 10337 leader_election.cc:288] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:26.327356 10304 raft_consensus.cc:2781] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:26.327951 10337 ts_tablet_manager.cc:1398] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20240501 08:13:26.328298 10304 raft_consensus.cc:686] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [term 1 LEADER]: Becoming Leader. State: Replica: c5ea0089ed5940089fb30561112e965a, State: Running, Role: LEADER
I20240501 08:13:26.328516 10304 consensus_queue.cc:237] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } }
I20240501 08:13:26.329298 10336 leader_election.cc:288] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:26.329432 10340 raft_consensus.cc:2781] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:26.330806 9603 catalog_manager.cc:5484] T ea0c2646c6b045fa9b4c75c04526f51f P c5ea0089ed5940089fb30561112e965a reported cstate change: term changed from 0 to 1, leader changed from <none> to c5ea0089ed5940089fb30561112e965a (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c5ea0089ed5940089fb30561112e965a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "c5ea0089ed5940089fb30561112e965a" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 36189 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:26.332245 10340 raft_consensus.cc:686] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:26.332607 10340 consensus_queue.cc:237] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.335948 10336 ts_tablet_manager.cc:1398] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.013s user 0.009s sys 0.004s
I20240501 08:13:26.336316 10336 tablet_bootstrap.cc:492] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: Bootstrap starting.
I20240501 08:13:26.337894 10336 tablet_bootstrap.cc:654] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:26.338433 9604 catalog_manager.cc:5484] T dd5729108dc84d80a5dfe37a57298edc P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:26.339368 10336 tablet_bootstrap.cc:492] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: No bootstrap required, opened a new log
I20240501 08:13:26.339511 10336 ts_tablet_manager.cc:1367] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240501 08:13:26.340216 10336 raft_consensus.cc:348] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.340364 10336 raft_consensus.cc:374] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:26.340431 10336 raft_consensus.cc:729] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Initialized, Role: FOLLOWER
I20240501 08:13:26.340637 10336 consensus_queue.cc:260] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.340770 10336 raft_consensus.cc:388] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:26.340857 10336 raft_consensus.cc:482] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:26.340937 10336 raft_consensus.cc:3037] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:26.342849 10336 raft_consensus.cc:504] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.343039 10336 leader_election.cc:302] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f22575e178c54dccae785f81ceff1f04; no voters:
I20240501 08:13:26.343238 10336 leader_election.cc:288] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:26.343308 10344 raft_consensus.cc:2781] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:26.343434 10344 raft_consensus.cc:686] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [term 1 LEADER]: Becoming Leader. State: Replica: f22575e178c54dccae785f81ceff1f04, State: Running, Role: LEADER
I20240501 08:13:26.343720 10344 consensus_queue.cc:237] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } }
I20240501 08:13:26.344190 10336 ts_tablet_manager.cc:1398] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: Time spent starting tablet: real 0.005s user 0.004s sys 0.001s
W20240501 08:13:26.344990 9923 tablet.cc:2369] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:26.346599 9604 catalog_manager.cc:5484] T 997c9d486efb4dd08680b42268c5aef4 P f22575e178c54dccae785f81ceff1f04 reported cstate change: term changed from 0 to 1, leader changed from <none> to f22575e178c54dccae785f81ceff1f04 (127.28.1.130). New cstate: current_term: 1 leader_uuid: "f22575e178c54dccae785f81ceff1f04" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f22575e178c54dccae785f81ceff1f04" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 38835 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:26.820596 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:37067
--local_ip_for_outbound_sockets=127.28.1.131
--tserver_master_addrs=127.28.1.190:46033
--webserver_port=42383
--webserver_interface=127.28.1.131
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:13:27.146060 10370 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:27.146368 10370 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:27.146474 10370 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:27.153954 10370 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:27.154119 10370 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:27.154247 10370 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:13:27.163259 10370 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:37067
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=42383
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:27.164562 10370 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:27.166296 10370 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:27.177791 10376 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:27.178185 10375 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:27.178711 10378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:27.179404 10370 server_base.cc:1034] running on GCE node
I20240501 08:13:27.179987 10370 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:27.180953 10370 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:27.182139 10370 hybrid_clock.cc:648] HybridClock initialized: now 1714551207182085 us; error 43 us; skew 500 ppm
I20240501 08:13:27.182468 10370 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:27.184700 10370 webserver.cc:460] Webserver started at http://127.28.1.131:42383/ using document root <none> and password file <none>
I20240501 08:13:27.185401 10370 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:27.185526 10370 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:27.190609 10370 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240501 08:13:27.193387 10384 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:27.195000 10370 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240501 08:13:27.195147 10370 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "199ef6a56e714a3894a5acbf81644839"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:27.195950 10370 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:27.219069 10370 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:27.219936 10370 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:27.220258 10370 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:27.221197 10370 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:27.223045 10391 ts_tablet_manager.cc:522] Loading tablet metadata (0/1 complete)
I20240501 08:13:27.228243 10370 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240501 08:13:27.228351 10370 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.006s user 0.000s sys 0.000s
I20240501 08:13:27.228478 10370 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240501 08:13:27.231470 10370 ts_tablet_manager.cc:590] Registered 1 tablets
I20240501 08:13:27.231596 10370 ts_tablet_manager.cc:569] Time spent register tablets: real 0.003s user 0.000s sys 0.000s
I20240501 08:13:27.231926 10391 tablet_bootstrap.cc:492] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Bootstrap starting.
I20240501 08:13:27.251581 10391 log.cc:824] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:27.255187 10391 tablet_bootstrap.cc:492] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Bootstrap replayed 1/1 log segments. Stats: ops{read=1 overwritten=0 applied=1 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:27.255666 10391 tablet_bootstrap.cc:492] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Bootstrap complete.
I20240501 08:13:27.256983 10391 ts_tablet_manager.cc:1367] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Time spent bootstrapping tablet: real 0.025s user 0.010s sys 0.014s
I20240501 08:13:27.267802 10391 raft_consensus.cc:348] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:27.268817 10391 raft_consensus.cc:729] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 199ef6a56e714a3894a5acbf81644839, State: Initialized, Role: FOLLOWER
I20240501 08:13:27.269512 10391 consensus_queue.cc:260] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:27.269716 10391 raft_consensus.cc:388] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:27.269838 10391 raft_consensus.cc:482] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:27.269994 10391 raft_consensus.cc:3037] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:27.272639 10370 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:37067
I20240501 08:13:27.272739 10498 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:37067 every 8 connection(s)
I20240501 08:13:27.273917 10370 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:13:27.273975 10391 raft_consensus.cc:504] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:27.274609 10391 leader_election.cc:302] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 199ef6a56e714a3894a5acbf81644839; no voters:
I20240501 08:13:27.275620 10391 leader_election.cc:288] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [CANDIDATE]: Term 2 election: Requested vote from peers
I20240501 08:13:27.275698 10501 raft_consensus.cc:2781] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:27.277330 10391 ts_tablet_manager.cc:1398] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839: Time spent starting tablet: real 0.020s user 0.013s sys 0.008s
I20240501 08:13:27.279408 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10370
I20240501 08:13:27.280210 10501 raft_consensus.cc:686] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [term 2 LEADER]: Becoming Leader. State: Replica: 199ef6a56e714a3894a5acbf81644839, State: Running, Role: LEADER
I20240501 08:13:27.280416 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:40459
--local_ip_for_outbound_sockets=127.28.1.132
--tserver_master_addrs=127.28.1.190:46033
--webserver_port=45309
--webserver_interface=127.28.1.132
--builtin_ntp_servers=127.28.1.148:43027
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:27.280681 10501 consensus_queue.cc:237] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } }
I20240501 08:13:27.300177 10499 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:27.300482 10499 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:27.301338 10499 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:27.304028 9604 ts_manager.cc:194] Re-registered known tserver with Master: 199ef6a56e714a3894a5acbf81644839 (127.28.1.131:37067)
I20240501 08:13:27.304494 9604 catalog_manager.cc:5484] T dc2cfd97babd4e1d8a23233e05d2b9c4 P 199ef6a56e714a3894a5acbf81644839 reported cstate change: term changed from 1 to 2. New cstate: current_term: 2 leader_uuid: "199ef6a56e714a3894a5acbf81644839" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "199ef6a56e714a3894a5acbf81644839" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 37067 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:27.308660 9604 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:39755
I20240501 08:13:27.311167 10499 heartbeater.cc:503] Master 127.28.1.190:46033 was elected leader, sending a full tablet report...
W20240501 08:13:27.614578 10508 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:27.614881 10508 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:27.615024 10508 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:27.622483 10508 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:27.622625 10508 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:27.622786 10508 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:13:27.631978 10508 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:43027
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:40459
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=45309
--tserver_master_addrs=127.28.1.190:46033
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:27.633267 10508 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:27.635018 10508 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:27.646397 10517 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:27.646859 10516 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:27.647300 10519 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:27.647751 10508 server_base.cc:1034] running on GCE node
I20240501 08:13:27.648362 10508 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:27.649264 10508 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:27.650481 10508 hybrid_clock.cc:648] HybridClock initialized: now 1714551207650409 us; error 39 us; skew 500 ppm
I20240501 08:13:27.650791 10508 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:27.652952 10508 webserver.cc:460] Webserver started at http://127.28.1.132:45309/ using document root <none> and password file <none>
I20240501 08:13:27.653719 10508 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:27.653861 10508 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:27.658941 10508 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240501 08:13:27.661556 10525 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:27.662842 10508 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240501 08:13:27.662994 10508 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "efa14afe2d28444898677bc8c15b4dc4"
format_stamp: "Formatted at 2024-05-01 08:13:19 on dist-test-slave-jj4n"
I20240501 08:13:27.663761 10508 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:27.689312 10508 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:27.690111 10508 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:27.690383 10508 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:27.691290 10508 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:27.692770 10508 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:27.692844 10508 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:27.692966 10508 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:27.693025 10508 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:27.721755 10508 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:40459
I20240501 08:13:27.721787 10637 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:40459 every 8 connection(s)
I20240501 08:13:27.723148 10508 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:13:27.725315 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10508
I20240501 08:13:27.735733 10638 heartbeater.cc:347] Connected to a master server at 127.28.1.190:46033
I20240501 08:13:27.736055 10638 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:27.736822 10638 heartbeater.cc:511] Master 127.28.1.190:46033 requested a full tablet report, sending...
I20240501 08:13:27.738245 9604 ts_manager.cc:194] Re-registered known tserver with Master: efa14afe2d28444898677bc8c15b4dc4 (127.28.1.132:40459)
I20240501 08:13:27.739200 9604 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:60689
I20240501 08:13:28.205334 9726 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:13:28.205420 10433 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:28.215212 10573 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:28.224576 9856 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:13:28.387871 10433 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:28.393013 9726 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:13:28.400442 10573 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:28.405639 9856 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:13:28.639822 10433 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:28.648334 10573 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:28.649308 9726 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240501 08:13:28.659603 9856 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
12f90df9121740219a05e9a58a61d52c | 127.28.1.190:46033 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:43027 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+---------+----------+----------------+-----------------
199ef6a56e714a3894a5acbf81644839 | 127.28.1.131:37067 | HEALTHY | <none> | 1 | 0
c5ea0089ed5940089fb30561112e965a | 127.28.1.129:36189 | HEALTHY | <none> | 6 | 0
efa14afe2d28444898677bc8c15b4dc4 | 127.28.1.132:40459 | HEALTHY | <none> | 0 | 0
f22575e178c54dccae785f81ceff1f04 | 127.28.1.130:38835 | HEALTHY | <none> | 6 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 4
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 4 server(s) checked
local_ip_for_outbound_sockets | 127.28.1.129 | experimental | 127.28.1.129:36189
local_ip_for_outbound_sockets | 127.28.1.130 | experimental | 127.28.1.130:38835
local_ip_for_outbound_sockets | 127.28.1.131 | experimental | 127.28.1.131:37067
local_ip_for_outbound_sockets | 127.28.1.132 | experimental | 127.28.1.132:40459
never_fsync | true | unsafe,advanced | all 4 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 4 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 4 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 4 server(s) checked
server_dump_info_format | pb | hidden | all 4 server(s) checked
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.28.1.129:36189
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.28.1.130:38835
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.28.1.131:37067
server_dump_info_path | /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.28.1.132:40459
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.28.1.148:43027 | all 4 server(s) checked
time_source | builtin | all 4 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 5 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 1
Median | 6
Third Quartile | 6
Maximum | 6
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 4
Tables | 5
Tablets | 13
Replicas | 13
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240501 08:13:28.705262 28678 log_verifier.cc:126] Checking tablet 3684ee20d5a2468eb25c78adec1a61bc
I20240501 08:13:28.712442 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 3684ee20d5a2468eb25c78adec1a61bc
I20240501 08:13:28.712554 28678 log_verifier.cc:126] Checking tablet 4ab60424862b4f3484d7bae7e655beb4
I20240501 08:13:28.719880 28678 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 4ab60424862b4f3484d7bae7e655beb4
I20240501 08:13:28.720017 28678 log_verifier.cc:126] Checking tablet 6dbd39eba86a4c09839a9065220d6342
I20240501 08:13:28.729321 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 6dbd39eba86a4c09839a9065220d6342
I20240501 08:13:28.729446 28678 log_verifier.cc:126] Checking tablet 957112d49396418cb624aa2dd72562ce
I20240501 08:13:28.739058 28678 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 957112d49396418cb624aa2dd72562ce
I20240501 08:13:28.739174 28678 log_verifier.cc:126] Checking tablet 997c9d486efb4dd08680b42268c5aef4
I20240501 08:13:28.741539 10638 heartbeater.cc:503] Master 127.28.1.190:46033 was elected leader, sending a full tablet report...
I20240501 08:13:28.746388 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 997c9d486efb4dd08680b42268c5aef4
I20240501 08:13:28.746495 28678 log_verifier.cc:126] Checking tablet a876452417414e9cbb0dd80fc75a8c8b
I20240501 08:13:28.787338 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet a876452417414e9cbb0dd80fc75a8c8b
I20240501 08:13:28.787420 28678 log_verifier.cc:126] Checking tablet aa14875486264c67bc7e2cca9e6f1af7
I20240501 08:13:28.794457 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet aa14875486264c67bc7e2cca9e6f1af7
I20240501 08:13:28.794549 28678 log_verifier.cc:126] Checking tablet b056930091f64e908cb0a8285e478d67
I20240501 08:13:28.801692 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet b056930091f64e908cb0a8285e478d67
I20240501 08:13:28.801800 28678 log_verifier.cc:126] Checking tablet c2a09ad35b82460587ccbcbca618ca73
I20240501 08:13:28.809320 28678 log_verifier.cc:177] Verified matching terms for 2 ops in tablet c2a09ad35b82460587ccbcbca618ca73
I20240501 08:13:28.809439 28678 log_verifier.cc:126] Checking tablet c76dd2c00c21491387b5ae952534325f
I20240501 08:13:28.816500 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet c76dd2c00c21491387b5ae952534325f
I20240501 08:13:28.816612 28678 log_verifier.cc:126] Checking tablet dc2cfd97babd4e1d8a23233e05d2b9c4
I20240501 08:13:28.823697 28678 log_verifier.cc:177] Verified matching terms for 2 ops in tablet dc2cfd97babd4e1d8a23233e05d2b9c4
I20240501 08:13:28.823820 28678 log_verifier.cc:126] Checking tablet dd5729108dc84d80a5dfe37a57298edc
I20240501 08:13:28.831243 28678 log_verifier.cc:177] Verified matching terms for 2 ops in tablet dd5729108dc84d80a5dfe37a57298edc
I20240501 08:13:28.831360 28678 log_verifier.cc:126] Checking tablet ea0c2646c6b045fa9b4c75c04526f51f
I20240501 08:13:28.838433 28678 log_verifier.cc:177] Verified matching terms for 1 ops in tablet ea0c2646c6b045fa9b4c75c04526f51f
I20240501 08:13:28.839619 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 9664
I20240501 08:13:28.840880 9786 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:28.939028 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 9664
I20240501 08:13:28.954298 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 9795
I20240501 08:13:28.955312 9917 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:29.049129 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 9795
I20240501 08:13:29.064317 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10370
I20240501 08:13:29.070255 10493 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:29.157713 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10370
I20240501 08:13:29.172020 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10508
I20240501 08:13:29.177865 10632 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:29.261451 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10508
I20240501 08:13:29.275027 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 9574
I20240501 08:13:29.276094 9635 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:29.391657 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 9574
2024-05-01T08:13:29Z chronyd exiting
[ OK ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5 (12084 ms)
[----------] 1 test from RebalancerAndSingleReplicaTablets (12084 ms total)
[----------] 2 tests from LocationAwareRebalancingParamTest
[ RUN ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1
I20240501 08:13:29.447470 28678 test_util.cc:268] Using random seed: 264884231
I20240501 08:13:29.449421 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:13:29.449497 28678 ts_itest-base.cc:113] --------------
I20240501 08:13:29.449534 28678 ts_itest-base.cc:114] 4 tablet servers
I20240501 08:13:29.449579 28678 ts_itest-base.cc:115] 2 replicas per TS
I20240501 08:13:29.449612 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:13:29Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:13:29Z Disabled control of system clock
I20240501 08:13:29.493595 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:45181
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:45181
--location_mapping_cmd=/tmp/dist-test-taskVWMLaH/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240501 08:13:29.818460 10751 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240501 08:13:29.818815 10751 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:29.818919 10751 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:29.819015 10751 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:29.826431 10751 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:29.826570 10751 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:13:29.826848 10751 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:29.826917 10751 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:13:29.827008 10751 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:13:29.836194 10751 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskVWMLaH/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:45181
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:45181
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:29.837488 10751 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:29.839154 10751 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:29.849602 10756 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:29.850152 10759 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:29.850633 10757 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:29.851146 10751 server_base.cc:1034] running on GCE node
I20240501 08:13:29.852329 10751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:29.853821 10751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:29.855038 10751 hybrid_clock.cc:648] HybridClock initialized: now 1714551209854961 us; error 49 us; skew 500 ppm
I20240501 08:13:29.855351 10751 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:29.857656 10751 webserver.cc:460] Webserver started at http://127.28.1.190:46609/ using document root <none> and password file <none>
I20240501 08:13:29.858354 10751 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:29.858470 10751 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:29.858767 10751 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:29.861374 10751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "a4d2fb4321784d4c83e31137c57ca508"
format_stamp: "Formatted at 2024-05-01 08:13:29 on dist-test-slave-jj4n"
I20240501 08:13:29.862116 10751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "a4d2fb4321784d4c83e31137c57ca508"
format_stamp: "Formatted at 2024-05-01 08:13:29 on dist-test-slave-jj4n"
I20240501 08:13:29.867079 10751 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.004s
I20240501 08:13:29.870321 10765 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:29.871604 10751 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:29.871773 10751 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "a4d2fb4321784d4c83e31137c57ca508"
format_stamp: "Formatted at 2024-05-01 08:13:29 on dist-test-slave-jj4n"
I20240501 08:13:29.871938 10751 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:29.887584 10751 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:29.888461 10751 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:29.888742 10751 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:29.905756 10751 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:45181
I20240501 08:13:29.905787 10816 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:45181 every 8 connection(s)
I20240501 08:13:29.907120 10751 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:13:29.909912 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10751
I20240501 08:13:29.910183 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:13:29.911283 10817 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:29.923874 10817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508: Bootstrap starting.
I20240501 08:13:29.926919 10817 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:29.927897 10817 log.cc:824] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:29.930366 10817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508: No bootstrap required, opened a new log
I20240501 08:13:29.936973 10817 raft_consensus.cc:348] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } }
I20240501 08:13:29.937350 10817 raft_consensus.cc:374] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:29.937458 10817 raft_consensus.cc:729] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4d2fb4321784d4c83e31137c57ca508, State: Initialized, Role: FOLLOWER
I20240501 08:13:29.938068 10817 consensus_queue.cc:260] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } }
I20240501 08:13:29.938266 10817 raft_consensus.cc:388] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:29.938386 10817 raft_consensus.cc:482] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:29.938522 10817 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:29.940587 10817 raft_consensus.cc:504] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } }
I20240501 08:13:29.941087 10817 leader_election.cc:302] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: a4d2fb4321784d4c83e31137c57ca508; no voters:
I20240501 08:13:29.941663 10817 leader_election.cc:288] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:29.941816 10822 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:29.942796 10822 raft_consensus.cc:686] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [term 1 LEADER]: Becoming Leader. State: Replica: a4d2fb4321784d4c83e31137c57ca508, State: Running, Role: LEADER
I20240501 08:13:29.943414 10822 consensus_queue.cc:237] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } }
I20240501 08:13:29.943812 10817 sys_catalog.cc:564] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:13:29.946535 10824 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4d2fb4321784d4c83e31137c57ca508. Latest consensus state: current_term: 1 leader_uuid: "a4d2fb4321784d4c83e31137c57ca508" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } } }
I20240501 08:13:29.946502 10823 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "a4d2fb4321784d4c83e31137c57ca508" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a4d2fb4321784d4c83e31137c57ca508" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 45181 } } }
I20240501 08:13:29.946902 10824 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:29.946905 10823 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508 [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:29.953063 10830 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:13:29.958555 10830 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:13:29.967311 10830 catalog_manager.cc:1347] Generated new cluster ID: 1f591806c738440cbb27c13e07fa434b
I20240501 08:13:29.967456 10830 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:13:29.985333 10830 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:13:29.986533 10830 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:13:29.995157 10830 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a4d2fb4321784d4c83e31137c57ca508: Generated new TSK 0
I20240501 08:13:29.996004 10830 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:13:30.006450 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:13:30.356076 10841 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:30.356374 10841 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:30.356505 10841 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:30.363910 10841 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:30.364080 10841 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:30.364251 10841 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:13:30.373245 10841 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:30.374524 10841 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:30.376294 10841 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:30.387673 10846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:30.388147 10847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:30.388744 10849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:30.389302 10841 server_base.cc:1034] running on GCE node
I20240501 08:13:30.389905 10841 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:30.390877 10841 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:30.392035 10841 hybrid_clock.cc:648] HybridClock initialized: now 1714551210391975 us; error 57 us; skew 500 ppm
I20240501 08:13:30.392339 10841 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:30.394500 10841 webserver.cc:460] Webserver started at http://127.28.1.129:44575/ using document root <none> and password file <none>
I20240501 08:13:30.395242 10841 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:30.395351 10841 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:30.395640 10841 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:30.398255 10841 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.399016 10841 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.404172 10841 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
I20240501 08:13:30.407533 10855 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.408943 10841 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:30.409107 10841 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.409312 10841 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:30.433156 10841 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:30.434029 10841 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:30.434293 10841 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:30.435256 10841 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:30.436762 10841 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:30.436853 10841 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.436959 10841 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:30.437027 10841 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.465907 10841 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:33053
I20240501 08:13:30.465963 10967 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:33053 every 8 connection(s)
I20240501 08:13:30.467191 10841 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:13:30.472225 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10841
I20240501 08:13:30.472405 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:13:30.475378 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:30.480190 10968 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:30.480520 10968 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:30.481492 10968 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:30.535445 10782 ts_manager.cc:194] Registered new tserver with Master: c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:30.537909 10782 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:50021
W20240501 08:13:30.815182 10972 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:30.815485 10972 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:30.815586 10972 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:30.822985 10972 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:30.823117 10972 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:30.823242 10972 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:13:30.832252 10972 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:30.833516 10972 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:30.835181 10972 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:30.846953 10978 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:30.847517 10979 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:30.847990 10981 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:30.848627 10972 server_base.cc:1034] running on GCE node
I20240501 08:13:30.849226 10972 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:30.850193 10972 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:30.851482 10972 hybrid_clock.cc:648] HybridClock initialized: now 1714551210851421 us; error 43 us; skew 500 ppm
I20240501 08:13:30.851853 10972 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:30.854219 10972 webserver.cc:460] Webserver started at http://127.28.1.130:41241/ using document root <none> and password file <none>
I20240501 08:13:30.854969 10972 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:30.855073 10972 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:30.855373 10972 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:30.858039 10972 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "3b5daf0ff6564ffa8d73213596c3915f"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.858793 10972 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "3b5daf0ff6564ffa8d73213596c3915f"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.863884 10972 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.005s
I20240501 08:13:30.867197 10987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.868643 10972 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
I20240501 08:13:30.868800 10972 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "3b5daf0ff6564ffa8d73213596c3915f"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:30.868990 10972 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:30.897904 10972 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:30.898797 10972 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:30.899065 10972 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:30.900065 10972 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:30.901547 10972 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:30.901634 10972 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.901746 10972 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:30.901811 10972 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:30.931263 10972 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:46677
I20240501 08:13:30.931334 11099 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:46677 every 8 connection(s)
I20240501 08:13:30.932577 10972 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:13:30.938869 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 10972
I20240501 08:13:30.939093 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:13:30.942859 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:30.946403 11100 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:30.946707 11100 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:30.947572 11100 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:30.994897 10782 ts_manager.cc:194] Registered new tserver with Master: 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130:46677)
I20240501 08:13:30.995869 10782 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:47497
W20240501 08:13:31.275566 11103 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:31.276023 11103 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:31.276187 11103 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:31.283762 11103 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:31.283914 11103 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:31.284088 11103 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:13:31.293188 11103 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:31.294497 11103 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:31.296258 11103 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:31.308894 11112 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:31.309381 11110 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:31.309944 11109 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:31.310452 11103 server_base.cc:1034] running on GCE node
I20240501 08:13:31.311051 11103 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:31.312028 11103 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:31.313230 11103 hybrid_clock.cc:648] HybridClock initialized: now 1714551211313175 us; error 51 us; skew 500 ppm
I20240501 08:13:31.313582 11103 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:31.315804 11103 webserver.cc:460] Webserver started at http://127.28.1.131:34897/ using document root <none> and password file <none>
I20240501 08:13:31.316570 11103 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:31.316682 11103 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:31.316984 11103 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:31.319516 11103 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "549bd4093e854fc0b8f822b3b96491c9"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.320293 11103 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "549bd4093e854fc0b8f822b3b96491c9"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.325239 11103 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:31.328559 11118 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.329802 11103 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:31.329953 11103 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "549bd4093e854fc0b8f822b3b96491c9"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.330142 11103 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:31.344952 11103 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:31.345805 11103 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:31.346068 11103 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:31.347047 11103 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:31.348606 11103 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:31.348704 11103 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.348825 11103 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:31.348894 11103 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.378527 11103 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:34007
I20240501 08:13:31.378580 11230 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:34007 every 8 connection(s)
I20240501 08:13:31.379813 11103 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:13:31.385030 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 11103
I20240501 08:13:31.385217 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:13:31.388669 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:31.393132 11231 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:31.393486 11231 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:31.394474 11231 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:31.438099 10782 ts_manager.cc:194] Registered new tserver with Master: 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131:34007)
I20240501 08:13:31.439406 10782 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:49237
I20240501 08:13:31.541170 10968 heartbeater.cc:503] Master 127.28.1.190:45181 was elected leader, sending a full tablet report...
W20240501 08:13:31.717437 11234 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:31.717751 11234 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:31.717893 11234 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:31.725366 11234 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:31.725500 11234 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:31.725652 11234 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:13:31.734722 11234 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:31.736120 11234 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:31.737812 11234 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:31.749593 11240 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:31.750191 11241 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:31.750761 11243 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:31.751297 11234 server_base.cc:1034] running on GCE node
I20240501 08:13:31.751950 11234 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:31.752924 11234 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:31.754146 11234 hybrid_clock.cc:648] HybridClock initialized: now 1714551211754103 us; error 33 us; skew 500 ppm
I20240501 08:13:31.754465 11234 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:31.756745 11234 webserver.cc:460] Webserver started at http://127.28.1.132:40097/ using document root <none> and password file <none>
I20240501 08:13:31.757452 11234 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:31.757560 11234 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:31.757855 11234 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:31.760452 11234 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "9e4413f7cb3344058795f529a81bfb99"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.761198 11234 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "9e4413f7cb3344058795f529a81bfb99"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.766270 11234 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.000s sys 0.005s
I20240501 08:13:31.769850 11249 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.771253 11234 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:31.771415 11234 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "9e4413f7cb3344058795f529a81bfb99"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:31.771615 11234 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:31.784081 11234 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:31.784859 11234 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:31.785108 11234 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:31.785986 11234 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:31.787531 11234 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:31.787691 11234 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.787796 11234 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:31.787850 11234 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:31.816207 11234 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:45109
I20240501 08:13:31.816253 11361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:45109 every 8 connection(s)
I20240501 08:13:31.817438 11234 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:13:31.827072 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 11234
I20240501 08:13:31.827273 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:13:31.829409 11362 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:31.829718 11362 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:31.830539 11362 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:31.872313 10781 ts_manager.cc:194] Registered new tserver with Master: 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:31.873442 10781 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:35187
I20240501 08:13:31.880321 28678 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240501 08:13:31.896271 28678 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240501 08:13:31.896422 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10972
I20240501 08:13:31.904397 11095 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:31.983040 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10972
I20240501 08:13:31.996430 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 11103
I20240501 08:13:32.003789 11226 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:32.082216 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 11103
I20240501 08:13:32.876446 11362 heartbeater.cc:503] Master 127.28.1.190:45181 was elected leader, sending a full tablet report...
I20240501 08:13:35.847311 10781 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39424:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:13:35.854033 10781 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240501 08:13:35.881996 10902 tablet_service.cc:1467] Processing CreateTablet for tablet 4f90d8a9896845f490e09dd5efe1893e (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.881707 11296 tablet_service.cc:1467] Processing CreateTablet for tablet 4f90d8a9896845f490e09dd5efe1893e (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.881608 10901 tablet_service.cc:1467] Processing CreateTablet for tablet dc66337da6784bd8b00f2c8529d9c550 (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.881749 10903 tablet_service.cc:1467] Processing CreateTablet for tablet 0dda417c185b428bb3456057d9ac2f4c (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.883288 11296 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f90d8a9896845f490e09dd5efe1893e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.883687 10901 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dc66337da6784bd8b00f2c8529d9c550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.884382 10903 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0dda417c185b428bb3456057d9ac2f4c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.884740 10902 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f90d8a9896845f490e09dd5efe1893e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.881608 11295 tablet_service.cc:1467] Processing CreateTablet for tablet dc66337da6784bd8b00f2c8529d9c550 (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.888468 11295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dc66337da6784bd8b00f2c8529d9c550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.892087 11297 tablet_service.cc:1467] Processing CreateTablet for tablet 0dda417c185b428bb3456057d9ac2f4c (DEFAULT_TABLE table=rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:35.892644 11297 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0dda417c185b428bb3456057d9ac2f4c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:35.902280 11384 tablet_bootstrap.cc:492] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:35.903936 11385 tablet_bootstrap.cc:492] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:35.905157 11384 tablet_bootstrap.cc:654] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.906469 11384 log.cc:824] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:35.907624 11385 tablet_bootstrap.cc:654] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.909224 11385 log.cc:824] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:35.910907 11384 tablet_bootstrap.cc:492] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:35.911196 11384 ts_tablet_manager.cc:1367] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.007s
I20240501 08:13:35.912685 11385 tablet_bootstrap.cc:492] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:35.912940 11385 ts_tablet_manager.cc:1367] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.004s
I20240501 08:13:35.919292 11385 raft_consensus.cc:348] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.919860 11385 raft_consensus.cc:374] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.920006 11385 raft_consensus.cc:729] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.920361 11384 raft_consensus.cc:348] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.920774 11384 raft_consensus.cc:374] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.920692 11385 consensus_queue.cc:260] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.920882 11384 raft_consensus.cc:729] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.921602 11384 consensus_queue.cc:260] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.922425 11385 ts_tablet_manager.cc:1398] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.009s user 0.007s sys 0.002s
I20240501 08:13:35.923282 11384 ts_tablet_manager.cc:1398] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.012s user 0.008s sys 0.003s
I20240501 08:13:35.923723 11384 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:35.925184 11384 tablet_bootstrap.cc:654] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.925272 11385 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:35.927001 11385 tablet_bootstrap.cc:654] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.930701 11385 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:35.930871 11385 ts_tablet_manager.cc:1367] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.000s
I20240501 08:13:35.931656 11385 raft_consensus.cc:348] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.931826 11385 raft_consensus.cc:374] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.931914 11385 raft_consensus.cc:729] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.932217 11385 consensus_queue.cc:260] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.932698 11385 ts_tablet_manager.cc:1398] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:13:35.933007 11385 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:35.933161 11384 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:35.933331 11384 ts_tablet_manager.cc:1367] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.006s
I20240501 08:13:35.934161 11384 raft_consensus.cc:348] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.934322 11384 raft_consensus.cc:374] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.934448 11384 raft_consensus.cc:729] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.934587 11385 tablet_bootstrap.cc:654] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.934731 11384 consensus_queue.cc:260] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.935310 11384 ts_tablet_manager.cc:1398] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:13:35.935616 11384 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:35.937304 11384 tablet_bootstrap.cc:654] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:35.938288 11385 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:35.938504 11385 ts_tablet_manager.cc:1367] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.001s
I20240501 08:13:35.939137 11385 raft_consensus.cc:348] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.939318 11385 raft_consensus.cc:374] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.939333 11384 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:35.939393 11385 raft_consensus.cc:729] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.939486 11384 ts_tablet_manager.cc:1367] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:13:35.939640 11385 consensus_queue.cc:260] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.940125 11384 raft_consensus.cc:348] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.940227 11385 ts_tablet_manager.cc:1398] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:13:35.940302 11384 raft_consensus.cc:374] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:35.940375 11384 raft_consensus.cc:729] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:35.940706 11384 consensus_queue.cc:260] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.941277 11384 ts_tablet_manager.cc:1398] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:13:35.942907 11388 raft_consensus.cc:482] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:35.943158 11388 raft_consensus.cc:504] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.944960 11388 leader_election.cc:288] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:35.951189 10923 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dc66337da6784bd8b00f2c8529d9c550" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" is_pre_election: true
I20240501 08:13:35.951689 10923 raft_consensus.cc:2454] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 0.
I20240501 08:13:35.952319 11253 leader_election.cc:302] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:35.952720 11388 raft_consensus.cc:2781] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:35.952916 11388 raft_consensus.cc:482] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:35.953044 11388 raft_consensus.cc:3037] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:35.955085 11388 raft_consensus.cc:504] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.955615 11388 leader_election.cc:288] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Requested vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:35.956220 10923 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dc66337da6784bd8b00f2c8529d9c550" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:35.956394 10923 raft_consensus.cc:3037] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:35.958422 10923 raft_consensus.cc:2454] T dc66337da6784bd8b00f2c8529d9c550 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 1.
I20240501 08:13:35.958850 11253 leader_election.cc:302] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:35.959174 11388 raft_consensus.cc:2781] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:35.959671 11388 raft_consensus.cc:686] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Becoming Leader. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Running, Role: LEADER
I20240501 08:13:35.960192 11388 consensus_queue.cc:237] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:35.964965 10782 catalog_manager.cc:5484] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132). New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: UNKNOWN } } }
W20240501 08:13:35.972862 10969 tablet.cc:2369] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:35.990656 11388 raft_consensus.cc:482] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:35.990919 11388 raft_consensus.cc:504] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.991468 11388 leader_election.cc:288] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:35.992134 10923 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0dda417c185b428bb3456057d9ac2f4c" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" is_pre_election: true
I20240501 08:13:35.992380 10923 raft_consensus.cc:2454] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 0.
I20240501 08:13:35.992766 11253 leader_election.cc:302] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:35.993059 11388 raft_consensus.cc:2781] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:35.993146 11388 raft_consensus.cc:482] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:35.993219 11388 raft_consensus.cc:3037] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:35.995178 11388 raft_consensus.cc:504] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:35.995704 11388 leader_election.cc:288] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Requested vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:35.996248 10923 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0dda417c185b428bb3456057d9ac2f4c" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:35.996440 10923 raft_consensus.cc:3037] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:35.998243 10923 raft_consensus.cc:2454] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 1.
I20240501 08:13:35.998646 11253 leader_election.cc:302] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:35.998942 11388 raft_consensus.cc:2781] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:35.999101 11388 raft_consensus.cc:686] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Becoming Leader. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Running, Role: LEADER
I20240501 08:13:35.999380 11388 consensus_queue.cc:237] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:36.001925 10782 catalog_manager.cc:5484] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132). New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } }
W20240501 08:13:36.071900 11363 tablet.cc:2369] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:36.377149 11389 raft_consensus.cc:482] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:36.377353 11389 raft_consensus.cc:504] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:36.378779 11389 leader_election.cc:288] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:36.385190 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f90d8a9896845f490e09dd5efe1893e" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:36.385565 11317 raft_consensus.cc:2454] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 0.
I20240501 08:13:36.386085 10856 leader_election.cc:302] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:36.386489 11389 raft_consensus.cc:2781] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:36.386605 11389 raft_consensus.cc:482] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:36.386682 11389 raft_consensus.cc:3037] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:36.388648 11389 raft_consensus.cc:504] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:36.389149 11389 leader_election.cc:288] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:36.389722 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f90d8a9896845f490e09dd5efe1893e" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:36.389889 11317 raft_consensus.cc:3037] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:36.391832 11317 raft_consensus.cc:2454] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 1.
I20240501 08:13:36.392261 10856 leader_election.cc:302] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:36.392588 11389 raft_consensus.cc:2781] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:36.393007 11389 raft_consensus.cc:686] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Becoming Leader. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:36.393500 11389 consensus_queue.cc:237] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:36.397766 10782 catalog_manager.cc:5484] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: term changed from 0 to 1, leader changed from <none> to c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:36.523764 11403 consensus_queue.cc:1035] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:36.560155 11405 consensus_queue.cc:1035] T dc66337da6784bd8b00f2c8529d9c550 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:13:36.784929 11402 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:36.785308 11402 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:36.792733 11402 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240501 08:13:36.843231 11389 consensus_queue.cc:1035] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 171.228 ms
time per row: 0.085614 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:13:37.964143 10781 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39424:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:13:37.964871 10781 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240501 08:13:37.972729 11296 tablet_service.cc:1467] Processing CreateTablet for tablet d07456dce1ed4c3385ee13efa386d8e3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.972947 10901 tablet_service.cc:1467] Processing CreateTablet for tablet d07456dce1ed4c3385ee13efa386d8e3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.972944 11297 tablet_service.cc:1467] Processing CreateTablet for tablet de54e034d6a84000b5678a7c0cf248ad (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.973245 11296 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d07456dce1ed4c3385ee13efa386d8e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.973327 10901 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d07456dce1ed4c3385ee13efa386d8e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.973505 11297 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de54e034d6a84000b5678a7c0cf248ad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.974138 10903 tablet_service.cc:1467] Processing CreateTablet for tablet de54e034d6a84000b5678a7c0cf248ad (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.974488 10903 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de54e034d6a84000b5678a7c0cf248ad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.979956 10902 tablet_service.cc:1467] Processing CreateTablet for tablet 45996f06656a48fc9f8580abfedd7b52 (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.981056 10902 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45996f06656a48fc9f8580abfedd7b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.980945 11295 tablet_service.cc:1467] Processing CreateTablet for tablet 45996f06656a48fc9f8580abfedd7b52 (DEFAULT_TABLE table=rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:37.981323 11295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45996f06656a48fc9f8580abfedd7b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:37.983330 11457 tablet_bootstrap.cc:492] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:37.985118 11457 tablet_bootstrap.cc:654] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:37.988046 11457 tablet_bootstrap.cc:492] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:37.988189 11458 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:37.988251 11457 ts_tablet_manager.cc:1367] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:37.988998 11457 raft_consensus.cc:348] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.989205 11457 raft_consensus.cc:374] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:37.989295 11457 raft_consensus.cc:729] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:37.989521 11457 consensus_queue.cc:260] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.989712 11458 tablet_bootstrap.cc:654] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:37.990316 11457 ts_tablet_manager.cc:1398] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:37.990876 11457 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:37.991997 11458 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:37.992152 11458 ts_tablet_manager.cc:1367] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.001s
I20240501 08:13:37.992451 11457 tablet_bootstrap.cc:654] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:37.992869 11458 raft_consensus.cc:348] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.993093 11458 raft_consensus.cc:374] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:37.993230 11458 raft_consensus.cc:729] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:37.993526 11458 consensus_queue.cc:260] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.994304 11457 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:37.994467 11457 ts_tablet_manager.cc:1367] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:13:37.995527 11457 raft_consensus.cc:348] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.995744 11457 raft_consensus.cc:374] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:37.995819 11457 raft_consensus.cc:729] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:37.996183 11457 consensus_queue.cc:260] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:37.996485 11458 ts_tablet_manager.cc:1398] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.004s user 0.002s sys 0.002s
I20240501 08:13:37.996730 11457 ts_tablet_manager.cc:1398] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:37.996866 11458 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:37.997011 11457 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:37.998381 11457 tablet_bootstrap.cc:654] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:37.998404 11458 tablet_bootstrap.cc:654] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.005033 11457 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:38.005205 11457 ts_tablet_manager.cc:1367] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.006s
I20240501 08:13:38.005370 11458 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:38.005535 11458 ts_tablet_manager.cc:1367] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.004s
I20240501 08:13:38.005988 11457 raft_consensus.cc:348] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.006192 11457 raft_consensus.cc:374] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.006294 11457 raft_consensus.cc:729] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.006229 11458 raft_consensus.cc:348] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.006412 11458 raft_consensus.cc:374] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.006489 11458 raft_consensus.cc:729] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.006537 11457 consensus_queue.cc:260] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.006708 11458 consensus_queue.cc:260] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.007050 11457 ts_tablet_manager.cc:1398] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:13:38.007207 11458 ts_tablet_manager.cc:1398] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:13:38.007464 11458 tablet_bootstrap.cc:492] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:38.008826 11458 tablet_bootstrap.cc:654] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.010591 11458 tablet_bootstrap.cc:492] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:38.010736 11458 ts_tablet_manager.cc:1367] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:38.011519 11458 raft_consensus.cc:348] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.011684 11458 raft_consensus.cc:374] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.011754 11458 raft_consensus.cc:729] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.011924 11458 consensus_queue.cc:260] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.012321 11458 ts_tablet_manager.cc:1398] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240501 08:13:38.055595 11409 raft_consensus.cc:482] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:38.055815 11409 raft_consensus.cc:504] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.056329 11409 leader_election.cc:288] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:38.056789 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de54e034d6a84000b5678a7c0cf248ad" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" is_pre_election: true
I20240501 08:13:38.057000 10922 raft_consensus.cc:2454] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 0.
I20240501 08:13:38.057413 11253 leader_election.cc:302] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.057715 11409 raft_consensus.cc:2781] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:38.057837 11409 raft_consensus.cc:482] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:38.057924 11409 raft_consensus.cc:3037] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.059885 11409 raft_consensus.cc:504] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.060364 11409 leader_election.cc:288] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Requested vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:38.060870 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de54e034d6a84000b5678a7c0cf248ad" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:38.061035 10922 raft_consensus.cc:3037] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.062959 10922 raft_consensus.cc:2454] T de54e034d6a84000b5678a7c0cf248ad P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 1.
I20240501 08:13:38.063313 11253 leader_election.cc:302] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.063568 11409 raft_consensus.cc:2781] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:38.063696 11409 raft_consensus.cc:686] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Becoming Leader. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Running, Role: LEADER
I20240501 08:13:38.063992 11409 consensus_queue.cc:237] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.066162 10781 catalog_manager.cc:5484] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132). New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:38.130700 11409 raft_consensus.cc:482] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:38.130973 11409 raft_consensus.cc:504] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.131582 11409 leader_election.cc:288] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:38.132299 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d07456dce1ed4c3385ee13efa386d8e3" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" is_pre_election: true
I20240501 08:13:38.132588 10922 raft_consensus.cc:2454] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 0.
I20240501 08:13:38.133038 11253 leader_election.cc:302] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.133394 11409 raft_consensus.cc:2781] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:38.133527 11409 raft_consensus.cc:482] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:38.133651 11409 raft_consensus.cc:3037] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.136101 11409 raft_consensus.cc:504] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.136592 11409 leader_election.cc:288] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Requested vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:38.137125 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d07456dce1ed4c3385ee13efa386d8e3" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:38.137383 10922 raft_consensus.cc:3037] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.140092 10922 raft_consensus.cc:2454] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 1.
I20240501 08:13:38.140528 11253 leader_election.cc:302] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.140796 11409 raft_consensus.cc:2781] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:38.140951 11409 raft_consensus.cc:686] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Becoming Leader. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Running, Role: LEADER
I20240501 08:13:38.141208 11409 consensus_queue.cc:237] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.143656 10781 catalog_manager.cc:5484] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132). New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:38.188882 11462 raft_consensus.cc:482] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:38.189067 11462 raft_consensus.cc:504] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.189630 11462 leader_election.cc:288] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:38.190239 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45996f06656a48fc9f8580abfedd7b52" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:38.190466 11317 raft_consensus.cc:2454] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 0.
I20240501 08:13:38.190898 10856 leader_election.cc:302] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.191174 11462 raft_consensus.cc:2781] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:38.191290 11462 raft_consensus.cc:482] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:38.191372 11462 raft_consensus.cc:3037] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.193300 11462 raft_consensus.cc:504] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.193755 11462 leader_election.cc:288] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:38.194257 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45996f06656a48fc9f8580abfedd7b52" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:38.194451 11317 raft_consensus.cc:3037] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.196334 11317 raft_consensus.cc:2454] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 1.
I20240501 08:13:38.196751 10856 leader_election.cc:302] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.197046 11462 raft_consensus.cc:2781] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:38.197391 11462 raft_consensus.cc:686] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Becoming Leader. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:38.197661 11462 consensus_queue.cc:237] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.199855 10781 catalog_manager.cc:5484] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: term changed from 0 to 1, leader changed from <none> to c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } }
W20240501 08:13:38.545585 11471 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:38.545931 11471 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:38.553385 11471 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240501 08:13:38.618115 11409 consensus_queue.cc:1035] T de54e034d6a84000b5678a7c0cf248ad P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:38.689146 11462 consensus_queue.cc:1035] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:38.708823 10922 raft_consensus.cc:1261] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240501 08:13:38.710379 11409 consensus_queue.cc:1035] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:38.711072 11496 mvcc.cc:205] Tried to move back new op lower bound from 7022801791741116416 to 7022801789739294720. Current Snapshot: MvccSnapshot[applied={T|T < 7022801791741116416}]
I20240501 08:13:38.722785 11505 mvcc.cc:205] Tried to move back new op lower bound from 7022801791741116416 to 7022801789739294720. Current Snapshot: MvccSnapshot[applied={T|T < 7022801791741116416}]
INSERT report
rows total: 2000
time total: 171.377 ms
time per row: 0.0856884 ms
I20240501 08:13:38.877218 10781 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39424:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:13:38.877918 10781 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240501 08:13:38.886051 10901 tablet_service.cc:1467] Processing CreateTablet for tablet 91c779081e204df4bc94654b880033db (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.886471 10903 tablet_service.cc:1467] Processing CreateTablet for tablet 7196efb1a1d24a25bfcc66df2c1a52a4 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.886613 10901 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91c779081e204df4bc94654b880033db. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.886893 10903 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7196efb1a1d24a25bfcc66df2c1a52a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.886785 10902 tablet_service.cc:1467] Processing CreateTablet for tablet 06d8b52123f641ce9d3b466f48726c75 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.887172 10902 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06d8b52123f641ce9d3b466f48726c75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.887832 11295 tablet_service.cc:1467] Processing CreateTablet for tablet 91c779081e204df4bc94654b880033db (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.888274 11295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91c779081e204df4bc94654b880033db. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.888319 11296 tablet_service.cc:1467] Processing CreateTablet for tablet 06d8b52123f641ce9d3b466f48726c75 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.888689 11296 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06d8b52123f641ce9d3b466f48726c75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.891340 11297 tablet_service.cc:1467] Processing CreateTablet for tablet 7196efb1a1d24a25bfcc66df2c1a52a4 (DEFAULT_TABLE table=rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:38.891716 11297 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7196efb1a1d24a25bfcc66df2c1a52a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:38.894956 11525 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:38.897061 11525 tablet_bootstrap.cc:654] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.898520 11524 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:38.898983 11525 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:38.899188 11525 ts_tablet_manager.cc:1367] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.004s
I20240501 08:13:38.900007 11525 raft_consensus.cc:348] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.900233 11525 raft_consensus.cc:374] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.900336 11525 raft_consensus.cc:729] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.900408 11524 tablet_bootstrap.cc:654] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.900601 11525 consensus_queue.cc:260] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.901228 11525 ts_tablet_manager.cc:1398] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240501 08:13:38.903324 11525 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:38.905078 11525 tablet_bootstrap.cc:654] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.908165 11524 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:38.908236 11525 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:38.908303 11524 ts_tablet_manager.cc:1367] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20240501 08:13:38.908376 11525 ts_tablet_manager.cc:1367] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:13:38.908944 11524 raft_consensus.cc:348] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.909106 11524 raft_consensus.cc:374] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.909046 11525 raft_consensus.cc:348] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.909189 11524 raft_consensus.cc:729] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.909240 11525 raft_consensus.cc:374] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.909307 11525 raft_consensus.cc:729] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.909442 11524 consensus_queue.cc:260] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.909518 11525 consensus_queue.cc:260] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.909992 11524 ts_tablet_manager.cc:1398] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:38.910085 11525 ts_tablet_manager.cc:1398] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:38.910244 11524 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:38.910344 11525 tablet_bootstrap.cc:492] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99: Bootstrap starting.
I20240501 08:13:38.911824 11525 tablet_bootstrap.cc:654] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.912142 11524 tablet_bootstrap.cc:654] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.919118 11524 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:38.919288 11524 ts_tablet_manager.cc:1367] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.008s
I20240501 08:13:38.919697 11525 tablet_bootstrap.cc:492] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99: No bootstrap required, opened a new log
I20240501 08:13:38.919842 11525 ts_tablet_manager.cc:1367] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.004s
I20240501 08:13:38.919901 11524 raft_consensus.cc:348] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.920120 11524 raft_consensus.cc:374] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.920194 11524 raft_consensus.cc:729] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.920420 11524 consensus_queue.cc:260] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.920534 11525 raft_consensus.cc:348] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.920722 11525 raft_consensus.cc:374] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.920791 11525 raft_consensus.cc:729] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e4413f7cb3344058795f529a81bfb99, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.920871 11524 ts_tablet_manager.cc:1398] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:13:38.921016 11525 consensus_queue.cc:260] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.921157 11524 tablet_bootstrap.cc:492] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2: Bootstrap starting.
I20240501 08:13:38.921557 11525 ts_tablet_manager.cc:1398] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:13:38.922652 11524 tablet_bootstrap.cc:654] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:38.925490 11524 tablet_bootstrap.cc:492] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2: No bootstrap required, opened a new log
I20240501 08:13:38.925626 11524 ts_tablet_manager.cc:1367] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:38.926303 11524 raft_consensus.cc:348] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.926492 11524 raft_consensus.cc:374] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:38.926584 11524 raft_consensus.cc:729] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Initialized, Role: FOLLOWER
I20240501 08:13:38.926795 11524 consensus_queue.cc:260] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:38.927244 11524 ts_tablet_manager.cc:1398] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:13:38.976433 11512 raft_consensus.cc:482] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:38.976631 11512 raft_consensus.cc:504] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.977248 11512 leader_election.cc:288] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:38.977809 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91c779081e204df4bc94654b880033db" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:38.978044 11317 raft_consensus.cc:2454] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 0.
I20240501 08:13:38.978416 10856 leader_election.cc:302] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.978665 11512 raft_consensus.cc:2781] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:38.978771 11512 raft_consensus.cc:482] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:38.978863 11512 raft_consensus.cc:3037] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.980839 11512 raft_consensus.cc:504] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.981271 11512 leader_election.cc:288] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:38.981804 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "91c779081e204df4bc94654b880033db" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:38.982002 11317 raft_consensus.cc:3037] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:38.983724 11317 raft_consensus.cc:2454] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 1.
I20240501 08:13:38.984143 10856 leader_election.cc:302] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:38.984426 11512 raft_consensus.cc:2781] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:38.984578 11512 raft_consensus.cc:686] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Becoming Leader. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:38.984840 11512 consensus_queue.cc:237] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } }
I20240501 08:13:38.987288 10781 catalog_manager.cc:5484] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: term changed from 0 to 1, leader changed from <none> to c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:39.113936 11466 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:39.114146 11466 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.114353 11512 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:39.114559 11512 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.114812 11466 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:39.115020 11512 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:39.115351 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" is_pre_election: true
I20240501 08:13:39.115463 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:39.115536 10922 raft_consensus.cc:2454] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9e4413f7cb3344058795f529a81bfb99 in term 0.
I20240501 08:13:39.115686 11317 raft_consensus.cc:2454] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 0.
I20240501 08:13:39.115926 11253 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:39.116194 11466 raft_consensus.cc:2781] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:39.116307 11466 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:39.116397 11466 raft_consensus.cc:3037] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:39.116417 10856 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:39.116703 11512 raft_consensus.cc:2781] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:39.116801 11512 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:39.116909 11512 raft_consensus.cc:3037] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:39.119043 11466 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.119550 11466 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Requested vote from peers c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:39.119458 11512 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.120075 11512 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:39.120074 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "9e4413f7cb3344058795f529a81bfb99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:39.120414 10922 raft_consensus.cc:2379] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 9e4413f7cb3344058795f529a81bfb99 in current term 1: Already voted for candidate c56f97cf6c5a4160bd5458de3568e5b2 in this term.
I20240501 08:13:39.120510 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:39.120764 11317 raft_consensus.cc:2379] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate c56f97cf6c5a4160bd5458de3568e5b2 in current term 1: Already voted for candidate 9e4413f7cb3344058795f529a81bfb99 in this term.
I20240501 08:13:39.120882 11253 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 2 responses out of 2 voters: 1 yes votes; 1 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99; no voters: c56f97cf6c5a4160bd5458de3568e5b2
I20240501 08:13:39.121131 11466 raft_consensus.cc:2726] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240501 08:13:39.121379 10856 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 2 responses out of 2 voters: 1 yes votes; 1 no votes. yes voters: c56f97cf6c5a4160bd5458de3568e5b2; no voters: 9e4413f7cb3344058795f529a81bfb99
I20240501 08:13:39.121598 11512 raft_consensus.cc:2726] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240501 08:13:39.175779 11512 raft_consensus.cc:482] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:39.176028 11512 raft_consensus.cc:504] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.176612 11512 leader_election.cc:288] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:39.177135 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:39.177367 11317 raft_consensus.cc:2454] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 0.
I20240501 08:13:39.177733 10856 leader_election.cc:302] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:39.177999 11512 raft_consensus.cc:2781] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:39.178100 11512 raft_consensus.cc:482] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:39.178176 11512 raft_consensus.cc:3037] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:39.180039 11512 raft_consensus.cc:504] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.180454 11512 leader_election.cc:288] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:39.180977 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:39.181167 11317 raft_consensus.cc:3037] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:39.182943 11317 raft_consensus.cc:2454] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 1.
I20240501 08:13:39.183292 10856 leader_election.cc:302] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:39.183542 11512 raft_consensus.cc:2781] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:39.183692 11512 raft_consensus.cc:686] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Becoming Leader. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:39.184006 11512 consensus_queue.cc:237] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:39.186364 10781 catalog_manager.cc:5484] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: term changed from 0 to 1, leader changed from <none> to c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } }
I20240501 08:13:39.536680 11512 consensus_queue.cc:1035] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:39.745245 11538 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:40.976271 11536 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:40.976459 11536 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:40.977022 11536 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:40.977633 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99" is_pre_election: true
I20240501 08:13:40.977856 11317 raft_consensus.cc:2454] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 1.
I20240501 08:13:40.978267 10856 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:40.978526 11536 raft_consensus.cc:2781] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240501 08:13:40.978638 11536 raft_consensus.cc:482] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:40.978715 11536 raft_consensus.cc:3037] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:40.980610 11536 raft_consensus.cc:504] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:40.981037 11536 leader_election.cc:288] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 2 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:40.981565 11317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7196efb1a1d24a25bfcc66df2c1a52a4" candidate_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:40.981740 11317 raft_consensus.cc:3037] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:40.983713 11317 raft_consensus.cc:2454] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c56f97cf6c5a4160bd5458de3568e5b2 in term 2.
I20240501 08:13:40.984205 10856 leader_election.cc:302] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 9e4413f7cb3344058795f529a81bfb99, c56f97cf6c5a4160bd5458de3568e5b2; no voters:
I20240501 08:13:40.984489 11536 raft_consensus.cc:2781] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:40.984829 11536 raft_consensus.cc:686] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 LEADER]: Becoming Leader. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:40.985061 11536 consensus_queue.cc:237] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } }
I20240501 08:13:40.987300 10781 catalog_manager.cc:5484] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: term changed from 0 to 2, leader changed from <none> to c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129). New cstate: current_term: 2 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } }
W20240501 08:13:41.335443 11546 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:41.335788 11546 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:41.344430 11546 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240501 08:13:41.411312 11317 raft_consensus.cc:1261] T 7196efb1a1d24a25bfcc66df2c1a52a4 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20240501 08:13:41.412163 11536 consensus_queue.cc:1035] T 7196efb1a1d24a25bfcc66df2c1a52a4 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 171.242 ms
time per row: 0.0856211 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:13:42.505264 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:46677
--local_ip_for_outbound_sockets=127.28.1.130
--tserver_master_addrs=127.28.1.190:45181
--webserver_port=41241
--webserver_interface=127.28.1.130
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:13:42.835335 11595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:42.835642 11595 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:42.835777 11595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:42.843593 11595 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:42.843744 11595 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:42.843899 11595 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:13:42.853080 11595 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:46677
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=41241
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:42.854408 11595 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:42.856140 11595 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:42.868113 11600 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:42.868695 11601 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:42.869118 11603 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:42.869643 11595 server_base.cc:1034] running on GCE node
I20240501 08:13:42.870258 11595 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:42.871345 11595 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:42.872557 11595 hybrid_clock.cc:648] HybridClock initialized: now 1714551222872489 us; error 63 us; skew 500 ppm
I20240501 08:13:42.872880 11595 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:42.879439 11595 webserver.cc:460] Webserver started at http://127.28.1.130:41241/ using document root <none> and password file <none>
I20240501 08:13:42.880271 11595 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:42.880403 11595 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:42.885727 11595 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:42.888641 11609 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:42.889999 11595 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:42.890158 11595 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "3b5daf0ff6564ffa8d73213596c3915f"
format_stamp: "Formatted at 2024-05-01 08:13:30 on dist-test-slave-jj4n"
I20240501 08:13:42.891111 11595 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:42.930351 11595 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:42.931221 11595 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:42.931509 11595 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:42.932528 11595 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:42.934170 11595 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:42.934242 11595 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:42.934357 11595 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:42.934410 11595 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:42.963465 11595 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:46677
I20240501 08:13:42.963517 11721 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:46677 every 8 connection(s)
I20240501 08:13:42.964951 11595 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:13:42.972519 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 11595
I20240501 08:13:42.973131 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:34007
--local_ip_for_outbound_sockets=127.28.1.131
--tserver_master_addrs=127.28.1.190:45181
--webserver_port=34897
--webserver_interface=127.28.1.131
--builtin_ntp_servers=127.28.1.148:40015
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:42.977916 11722 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:42.978228 11722 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:42.979100 11722 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:42.980971 10776 ts_manager.cc:194] Re-registered known tserver with Master: 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130:46677)
I20240501 08:13:42.982120 10776 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:36977
W20240501 08:13:43.300734 11726 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:43.301029 11726 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:43.301159 11726 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:43.308703 11726 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:43.308841 11726 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:43.308991 11726 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:13:43.318138 11726 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40015
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:34007
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=34897
--tserver_master_addrs=127.28.1.190:45181
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:43.319459 11726 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:43.321378 11726 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:43.333155 11731 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:43.333477 11732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:43.333930 11734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:43.334408 11726 server_base.cc:1034] running on GCE node
I20240501 08:13:43.335034 11726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:43.336086 11726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:43.337306 11726 hybrid_clock.cc:648] HybridClock initialized: now 1714551223337248 us; error 48 us; skew 500 ppm
I20240501 08:13:43.337632 11726 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:43.340126 11726 webserver.cc:460] Webserver started at http://127.28.1.131:34897/ using document root <none> and password file <none>
I20240501 08:13:43.340901 11726 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:43.341027 11726 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:43.346982 11726 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.006s sys 0.000s
I20240501 08:13:43.350345 11740 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:43.351838 11726 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:43.352048 11726 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "549bd4093e854fc0b8f822b3b96491c9"
format_stamp: "Formatted at 2024-05-01 08:13:31 on dist-test-slave-jj4n"
I20240501 08:13:43.352900 11726 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:43.378438 11726 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:43.379287 11726 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:43.379577 11726 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:43.380553 11726 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:43.382040 11726 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:43.382123 11726 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:43.382243 11726 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:43.382294 11726 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:43.411904 11726 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:34007
I20240501 08:13:43.411954 11852 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:34007 every 8 connection(s)
I20240501 08:13:43.413403 11726 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:13:43.422065 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 11726
I20240501 08:13:43.425961 11853 heartbeater.cc:347] Connected to a master server at 127.28.1.190:45181
I20240501 08:13:43.426316 11853 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:43.427311 11853 heartbeater.cc:511] Master 127.28.1.190:45181 requested a full tablet report, sending...
I20240501 08:13:43.428934 10776 ts_manager.cc:194] Re-registered known tserver with Master: 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131:34007)
I20240501 08:13:43.429723 10776 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:32961
I20240501 08:13:43.872889 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:43.890645 10903 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:43.893836 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:43.921705 11788 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:43.985423 11722 heartbeater.cc:503] Master 127.28.1.190:45181 was elected leader, sending a full tablet report...
I20240501 08:13:44.067782 11788 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.068789 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.073992 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:44.078476 10903 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:44.235855 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.239735 11788 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.244347 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:44.254040 10903 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:44.394819 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.396411 11788 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:44.399384 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:44.427301 10903 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:13:44.432821 11853 heartbeater.cc:503] Master 127.28.1.190:45181 was elected leader, sending a full tablet report...
I20240501 08:13:44.484097 10922 consensus_queue.cc:237] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:44.486274 11315 raft_consensus.cc:1261] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:13:44.486894 11540 consensus_queue.cc:1035] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.489777 11979 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
I20240501 08:13:44.490212 11315 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
W20240501 08:13:44.493556 10858 consensus_peers.cc:485] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 -> Peer 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130:46677): Couldn't send request to peer 3b5daf0ff6564ffa8d73213596c3915f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 91c779081e204df4bc94654b880033db. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.494978 10776 catalog_manager.cc:5484] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:44.508498 11315 consensus_queue.cc:237] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:44.510937 10922 raft_consensus.cc:1261] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:13:44.511569 11466 consensus_queue.cc:1035] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.514638 12002 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
I20240501 08:13:44.515308 10922 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
W20240501 08:13:44.518208 11250 consensus_peers.cc:485] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 -> Peer 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131:34007): Couldn't send request to peer 549bd4093e854fc0b8f822b3b96491c9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d07456dce1ed4c3385ee13efa386d8e3. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.520414 10774 catalog_manager.cc:5484] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:44.530867 10922 consensus_queue.cc:237] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:44.532851 11315 raft_consensus.cc:1261] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:13:44.533010 10858 consensus_peers.cc:485] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 -> Peer 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130:46677): Couldn't send request to peer 3b5daf0ff6564ffa8d73213596c3915f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4f90d8a9896845f490e09dd5efe1893e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.533320 11979 consensus_queue.cc:1035] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.535742 11540 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
I20240501 08:13:44.536324 11315 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
I20240501 08:13:44.539995 10776 catalog_manager.cc:5484] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:44.551247 11315 consensus_queue.cc:237] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:44.553406 10922 raft_consensus.cc:1261] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:13:44.553442 11250 consensus_peers.cc:485] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 -> Peer 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131:34007): Couldn't send request to peer 549bd4093e854fc0b8f822b3b96491c9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0dda417c185b428bb3456057d9ac2f4c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.553881 12002 consensus_queue.cc:1035] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.556459 11466 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
I20240501 08:13:44.557118 10922 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
I20240501 08:13:44.561127 10776 catalog_manager.cc:5484] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:44.573339 10922 consensus_queue.cc:237] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:44.575217 11315 raft_consensus.cc:1261] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240501 08:13:44.575410 10858 consensus_peers.cc:485] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 -> Peer 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130:46677): Couldn't send request to peer 3b5daf0ff6564ffa8d73213596c3915f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 45996f06656a48fc9f8580abfedd7b52. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.575821 11979 consensus_queue.cc:1035] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.578436 11540 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
I20240501 08:13:44.579039 11315 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } } }
I20240501 08:13:44.583703 10779 catalog_manager.cc:5484] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index -1 to 4, NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) added. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:44.593881 10922 consensus_queue.cc:237] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:44.595861 11315 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:13:44.596501 11979 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:13:44.599040 11540 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
I20240501 08:13:44.599610 11315 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
W20240501 08:13:44.602228 10856 consensus_peers.cc:485] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 -> Peer 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131:34007): Couldn't send request to peer 549bd4093e854fc0b8f822b3b96491c9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 06d8b52123f641ce9d3b466f48726c75. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:13:44.604300 10779 catalog_manager.cc:5484] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index -1 to 4, NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) added. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } } }
I20240501 08:13:44.872625 12073 ts_tablet_manager.cc:902] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Initiating tablet copy from peer c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:44.875172 12073 tablet_copy_client.cc:308] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:33053
I20240501 08:13:44.881784 10943 tablet_copy_service.cc:140] P c56f97cf6c5a4160bd5458de3568e5b2: Received BeginTabletCopySession request for tablet 91c779081e204df4bc94654b880033db from peer 3b5daf0ff6564ffa8d73213596c3915f ({username='slave'} at 127.28.1.130:43969)
I20240501 08:13:44.881999 10943 tablet_copy_service.cc:161] P c56f97cf6c5a4160bd5458de3568e5b2: Beginning new tablet copy session on tablet 91c779081e204df4bc94654b880033db from peer 3b5daf0ff6564ffa8d73213596c3915f at {username='slave'} at 127.28.1.130:43969: session id = 3b5daf0ff6564ffa8d73213596c3915f-91c779081e204df4bc94654b880033db
I20240501 08:13:44.884780 10943 tablet_copy_source_session.cc:205] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:44.890333 12073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 91c779081e204df4bc94654b880033db. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:44.903195 12073 tablet_copy_client.cc:783] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:44.903548 12073 tablet_copy_client.cc:647] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:44.906144 12073 tablet_copy_client.cc:523] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:44.909938 12073 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap starting.
I20240501 08:13:44.919334 12073 log.cc:824] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:44.954355 12073 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:44.954890 12073 tablet_bootstrap.cc:492] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap complete.
I20240501 08:13:44.955323 12073 ts_tablet_manager.cc:1367] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Time spent bootstrapping tablet: real 0.046s user 0.040s sys 0.004s
I20240501 08:13:44.960287 12073 raft_consensus.cc:348] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:44.960640 12073 raft_consensus.cc:729] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b5daf0ff6564ffa8d73213596c3915f, State: Initialized, Role: LEARNER
I20240501 08:13:44.961102 12073 consensus_queue.cc:260] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:44.962671 12073 ts_tablet_manager.cc:1398] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f: Time spent starting tablet: real 0.007s user 0.009s sys 0.000s
I20240501 08:13:44.963742 10943 tablet_copy_service.cc:342] P c56f97cf6c5a4160bd5458de3568e5b2: Request end of tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-91c779081e204df4bc94654b880033db received from {username='slave'} at 127.28.1.130:43969
I20240501 08:13:44.963912 10943 tablet_copy_service.cc:433] P c56f97cf6c5a4160bd5458de3568e5b2: ending tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-91c779081e204df4bc94654b880033db on tablet 91c779081e204df4bc94654b880033db with peer 3b5daf0ff6564ffa8d73213596c3915f
I20240501 08:13:45.063490 12073 ts_tablet_manager.cc:902] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Initiating tablet copy from peer c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:45.063982 12101 ts_tablet_manager.cc:902] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Initiating tablet copy from peer 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:45.064831 12073 tablet_copy_client.cc:308] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:33053
I20240501 08:13:45.065532 10943 tablet_copy_service.cc:140] P c56f97cf6c5a4160bd5458de3568e5b2: Received BeginTabletCopySession request for tablet 45996f06656a48fc9f8580abfedd7b52 from peer 3b5daf0ff6564ffa8d73213596c3915f ({username='slave'} at 127.28.1.130:43969)
I20240501 08:13:45.065725 10943 tablet_copy_service.cc:161] P c56f97cf6c5a4160bd5458de3568e5b2: Beginning new tablet copy session on tablet 45996f06656a48fc9f8580abfedd7b52 from peer 3b5daf0ff6564ffa8d73213596c3915f at {username='slave'} at 127.28.1.130:43969: session id = 3b5daf0ff6564ffa8d73213596c3915f-45996f06656a48fc9f8580abfedd7b52
I20240501 08:13:45.065749 12107 ts_tablet_manager.cc:902] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Initiating tablet copy from peer c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:45.067998 12107 tablet_copy_client.cc:308] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:33053
I20240501 08:13:45.068809 12101 tablet_copy_client.cc:308] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:45109
I20240501 08:13:45.068739 10942 tablet_copy_service.cc:140] P c56f97cf6c5a4160bd5458de3568e5b2: Received BeginTabletCopySession request for tablet 4f90d8a9896845f490e09dd5efe1893e from peer 3b5daf0ff6564ffa8d73213596c3915f ({username='slave'} at 127.28.1.130:43969)
I20240501 08:13:45.068957 10942 tablet_copy_service.cc:161] P c56f97cf6c5a4160bd5458de3568e5b2: Beginning new tablet copy session on tablet 4f90d8a9896845f490e09dd5efe1893e from peer 3b5daf0ff6564ffa8d73213596c3915f at {username='slave'} at 127.28.1.130:43969: session id = 3b5daf0ff6564ffa8d73213596c3915f-4f90d8a9896845f490e09dd5efe1893e
I20240501 08:13:45.069268 10943 tablet_copy_source_session.cc:205] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:45.070689 12073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45996f06656a48fc9f8580abfedd7b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:45.071280 10942 tablet_copy_source_session.cc:205] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:45.072469 12107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f90d8a9896845f490e09dd5efe1893e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:45.077970 12073 tablet_copy_client.cc:783] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:45.078207 12073 tablet_copy_client.cc:647] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:45.079092 12107 tablet_copy_client.cc:783] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:45.079303 12107 tablet_copy_client.cc:647] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:45.079471 11337 tablet_copy_service.cc:140] P 9e4413f7cb3344058795f529a81bfb99: Received BeginTabletCopySession request for tablet 0dda417c185b428bb3456057d9ac2f4c from peer 549bd4093e854fc0b8f822b3b96491c9 ({username='slave'} at 127.28.1.131:34201)
I20240501 08:13:45.079689 11337 tablet_copy_service.cc:161] P 9e4413f7cb3344058795f529a81bfb99: Beginning new tablet copy session on tablet 0dda417c185b428bb3456057d9ac2f4c from peer 549bd4093e854fc0b8f822b3b96491c9 at {username='slave'} at 127.28.1.131:34201: session id = 549bd4093e854fc0b8f822b3b96491c9-0dda417c185b428bb3456057d9ac2f4c
I20240501 08:13:45.080013 12073 tablet_copy_client.cc:523] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:45.081105 12107 tablet_copy_client.cc:523] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:45.083279 11337 tablet_copy_source_session.cc:205] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:45.083474 12073 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap starting.
I20240501 08:13:45.085075 12107 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap starting.
I20240501 08:13:45.089140 12101 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0dda417c185b428bb3456057d9ac2f4c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:45.091580 12116 ts_tablet_manager.cc:902] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Initiating tablet copy from peer 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109)
I20240501 08:13:45.102700 12101 tablet_copy_client.cc:783] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:45.103071 12101 tablet_copy_client.cc:647] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:45.105362 12101 tablet_copy_client.cc:523] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:45.110230 12101 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap starting.
I20240501 08:13:45.120366 12116 tablet_copy_client.cc:308] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:45109
I20240501 08:13:45.121204 11337 tablet_copy_service.cc:140] P 9e4413f7cb3344058795f529a81bfb99: Received BeginTabletCopySession request for tablet d07456dce1ed4c3385ee13efa386d8e3 from peer 549bd4093e854fc0b8f822b3b96491c9 ({username='slave'} at 127.28.1.131:34201)
I20240501 08:13:45.121407 11337 tablet_copy_service.cc:161] P 9e4413f7cb3344058795f529a81bfb99: Beginning new tablet copy session on tablet d07456dce1ed4c3385ee13efa386d8e3 from peer 549bd4093e854fc0b8f822b3b96491c9 at {username='slave'} at 127.28.1.131:34201: session id = 549bd4093e854fc0b8f822b3b96491c9-d07456dce1ed4c3385ee13efa386d8e3
I20240501 08:13:45.124269 11337 tablet_copy_source_session.cc:205] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:45.125502 12116 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d07456dce1ed4c3385ee13efa386d8e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:45.127329 12101 log.cc:824] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:45.130874 12123 ts_tablet_manager.cc:902] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Initiating tablet copy from peer c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:45.132823 12116 tablet_copy_client.cc:783] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:45.133142 12116 tablet_copy_client.cc:647] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:45.135131 12116 tablet_copy_client.cc:523] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:45.138530 12116 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap starting.
I20240501 08:13:45.142555 12073 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:45.142962 12073 tablet_bootstrap.cc:492] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap complete.
I20240501 08:13:45.143249 12073 ts_tablet_manager.cc:1367] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Time spent bootstrapping tablet: real 0.060s user 0.044s sys 0.008s
I20240501 08:13:45.144390 12073 raft_consensus.cc:348] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:45.144666 12073 raft_consensus.cc:729] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b5daf0ff6564ffa8d73213596c3915f, State: Initialized, Role: LEARNER
I20240501 08:13:45.144906 12073 consensus_queue.cc:260] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:45.148406 12073 ts_tablet_manager.cc:1398] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:45.151444 10942 tablet_copy_service.cc:342] P c56f97cf6c5a4160bd5458de3568e5b2: Request end of tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-45996f06656a48fc9f8580abfedd7b52 received from {username='slave'} at 127.28.1.130:43969
I20240501 08:13:45.151626 10942 tablet_copy_service.cc:433] P c56f97cf6c5a4160bd5458de3568e5b2: ending tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-45996f06656a48fc9f8580abfedd7b52 on tablet 45996f06656a48fc9f8580abfedd7b52 with peer 3b5daf0ff6564ffa8d73213596c3915f
I20240501 08:13:45.160347 12107 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:45.160813 12107 tablet_bootstrap.cc:492] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Bootstrap complete.
I20240501 08:13:45.161108 12107 ts_tablet_manager.cc:1367] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Time spent bootstrapping tablet: real 0.076s user 0.044s sys 0.015s
I20240501 08:13:45.161909 12107 raft_consensus.cc:348] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:45.162169 12107 raft_consensus.cc:729] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3b5daf0ff6564ffa8d73213596c3915f, State: Initialized, Role: LEARNER
I20240501 08:13:45.162381 12107 consensus_queue.cc:260] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: true } }
I20240501 08:13:45.165673 12107 ts_tablet_manager.cc:1398] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f: Time spent starting tablet: real 0.004s user 0.002s sys 0.001s
I20240501 08:13:45.169015 10942 tablet_copy_service.cc:342] P c56f97cf6c5a4160bd5458de3568e5b2: Request end of tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-4f90d8a9896845f490e09dd5efe1893e received from {username='slave'} at 127.28.1.130:43969
I20240501 08:13:45.169183 10942 tablet_copy_service.cc:433] P c56f97cf6c5a4160bd5458de3568e5b2: ending tablet copy session 3b5daf0ff6564ffa8d73213596c3915f-4f90d8a9896845f490e09dd5efe1893e on tablet 4f90d8a9896845f490e09dd5efe1893e with peer 3b5daf0ff6564ffa8d73213596c3915f
I20240501 08:13:45.170879 12123 tablet_copy_client.cc:308] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:33053
I20240501 08:13:45.191176 12101 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:45.191828 12101 tablet_bootstrap.cc:492] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap complete.
I20240501 08:13:45.192382 12101 ts_tablet_manager.cc:1367] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Time spent bootstrapping tablet: real 0.082s user 0.059s sys 0.016s
I20240501 08:13:45.193337 10942 tablet_copy_service.cc:140] P c56f97cf6c5a4160bd5458de3568e5b2: Received BeginTabletCopySession request for tablet 06d8b52123f641ce9d3b466f48726c75 from peer 549bd4093e854fc0b8f822b3b96491c9 ({username='slave'} at 127.28.1.131:59351)
I20240501 08:13:45.193516 10942 tablet_copy_service.cc:161] P c56f97cf6c5a4160bd5458de3568e5b2: Beginning new tablet copy session on tablet 06d8b52123f641ce9d3b466f48726c75 from peer 549bd4093e854fc0b8f822b3b96491c9 at {username='slave'} at 127.28.1.131:59351: session id = 549bd4093e854fc0b8f822b3b96491c9-06d8b52123f641ce9d3b466f48726c75
I20240501 08:13:45.196465 10942 tablet_copy_source_session.cc:205] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:13:45.197885 12123 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06d8b52123f641ce9d3b466f48726c75. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:45.199235 12101 raft_consensus.cc:348] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.199755 12101 raft_consensus.cc:729] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 549bd4093e854fc0b8f822b3b96491c9, State: Initialized, Role: LEARNER
I20240501 08:13:45.200265 12116 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:45.200374 12101 consensus_queue.cc:260] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.200723 12116 tablet_bootstrap.cc:492] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap complete.
I20240501 08:13:45.201001 12116 ts_tablet_manager.cc:1367] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Time spent bootstrapping tablet: real 0.063s user 0.050s sys 0.011s
I20240501 08:13:45.202180 12116 raft_consensus.cc:348] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.202438 12116 raft_consensus.cc:729] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 549bd4093e854fc0b8f822b3b96491c9, State: Initialized, Role: LEARNER
I20240501 08:13:45.202684 12116 consensus_queue.cc:260] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.206115 12123 tablet_copy_client.cc:783] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 0 data blocks...
I20240501 08:13:45.206459 12123 tablet_copy_client.cc:647] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Starting download of 1 WAL segments...
I20240501 08:13:45.208433 12123 tablet_copy_client.cc:523] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:13:45.214807 12123 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap starting.
I20240501 08:13:45.221519 12101 ts_tablet_manager.cc:1398] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9: Time spent starting tablet: real 0.029s user 0.017s sys 0.011s
I20240501 08:13:45.221524 12116 ts_tablet_manager.cc:1398] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9: Time spent starting tablet: real 0.020s user 0.005s sys 0.016s
I20240501 08:13:45.223230 11337 tablet_copy_service.cc:342] P 9e4413f7cb3344058795f529a81bfb99: Request end of tablet copy session 549bd4093e854fc0b8f822b3b96491c9-0dda417c185b428bb3456057d9ac2f4c received from {username='slave'} at 127.28.1.131:34201
I20240501 08:13:45.223412 11337 tablet_copy_service.cc:433] P 9e4413f7cb3344058795f529a81bfb99: ending tablet copy session 549bd4093e854fc0b8f822b3b96491c9-0dda417c185b428bb3456057d9ac2f4c on tablet 0dda417c185b428bb3456057d9ac2f4c with peer 549bd4093e854fc0b8f822b3b96491c9
I20240501 08:13:45.224632 11336 tablet_copy_service.cc:342] P 9e4413f7cb3344058795f529a81bfb99: Request end of tablet copy session 549bd4093e854fc0b8f822b3b96491c9-d07456dce1ed4c3385ee13efa386d8e3 received from {username='slave'} at 127.28.1.131:34201
I20240501 08:13:45.224776 11336 tablet_copy_service.cc:433] P 9e4413f7cb3344058795f529a81bfb99: ending tablet copy session 549bd4093e854fc0b8f822b3b96491c9-d07456dce1ed4c3385ee13efa386d8e3 on tablet d07456dce1ed4c3385ee13efa386d8e3 with peer 549bd4093e854fc0b8f822b3b96491c9
I20240501 08:13:45.259922 12123 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:13:45.260499 12123 tablet_bootstrap.cc:492] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Bootstrap complete.
I20240501 08:13:45.260845 12123 ts_tablet_manager.cc:1367] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Time spent bootstrapping tablet: real 0.046s user 0.030s sys 0.013s
I20240501 08:13:45.261659 12123 raft_consensus.cc:348] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.261900 12123 raft_consensus.cc:729] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 549bd4093e854fc0b8f822b3b96491c9, State: Initialized, Role: LEARNER
I20240501 08:13:45.262096 12123 consensus_queue.cc:260] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: true } }
I20240501 08:13:45.262722 12123 ts_tablet_manager.cc:1398] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:13:45.263520 10942 tablet_copy_service.cc:342] P c56f97cf6c5a4160bd5458de3568e5b2: Request end of tablet copy session 549bd4093e854fc0b8f822b3b96491c9-06d8b52123f641ce9d3b466f48726c75 received from {username='slave'} at 127.28.1.131:59351
I20240501 08:13:45.263653 10942 tablet_copy_service.cc:433] P c56f97cf6c5a4160bd5458de3568e5b2: ending tablet copy session 549bd4093e854fc0b8f822b3b96491c9-06d8b52123f641ce9d3b466f48726c75 on tablet 06d8b52123f641ce9d3b466f48726c75 with peer 549bd4093e854fc0b8f822b3b96491c9
I20240501 08:13:45.373064 11677 raft_consensus.cc:1203] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.481205 11677 raft_consensus.cc:1203] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.551889 11808 raft_consensus.cc:1203] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.591701 11677 raft_consensus.cc:1203] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.644194 11808 raft_consensus.cc:1203] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.707082 11808 raft_consensus.cc:1203] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240501 08:13:45.873268 12164 raft_consensus.cc:1050] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2: attempting to promote NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f to VOTER
I20240501 08:13:45.874051 12164 consensus_queue.cc:237] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:45.876103 11315 raft_consensus.cc:1261] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:45.876166 11677 raft_consensus.cc:1261] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:45.876718 11979 consensus_queue.cc:1035] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:45.877035 12165 consensus_queue.cc:1035] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:45.879547 11979 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.881420 11677 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.882270 11315 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.885182 10776 catalog_manager.cc:5484] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:45.891341 10922 consensus_queue.cc:237] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:45.893117 11677 raft_consensus.cc:1261] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:45.893586 12164 consensus_queue.cc:1035] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:45.895478 12165 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.896132 11677 raft_consensus.cc:2932] T 91c779081e204df4bc94654b880033db P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.899019 10769 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 91c779081e204df4bc94654b880033db with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:45.900394 10776 catalog_manager.cc:5484] T 91c779081e204df4bc94654b880033db P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:45.903865 11296 tablet_service.cc:1514] Processing DeleteTablet for tablet 91c779081e204df4bc94654b880033db with delete_type TABLET_DATA_TOMBSTONED (TS 9e4413f7cb3344058795f529a81bfb99 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44162
I20240501 08:13:45.904700 12235 tablet_replica.cc:330] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: stopping tablet replica
I20240501 08:13:45.905067 12235 raft_consensus.cc:2229] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:45.905483 12235 raft_consensus.cc:2258] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:45.906813 12235 ts_tablet_manager.cc:1875] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:45.911418 12235 ts_tablet_manager.cc:1888] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:45.911553 12235 log.cc:1196] T 91c779081e204df4bc94654b880033db P 9e4413f7cb3344058795f529a81bfb99: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/91c779081e204df4bc94654b880033db
I20240501 08:13:45.912542 10766 catalog_manager.cc:4830] TS 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109): tablet 91c779081e204df4bc94654b880033db (table rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]) successfully deleted
I20240501 08:13:45.993048 12165 raft_consensus.cc:1050] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2: attempting to promote NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f to VOTER
I20240501 08:13:45.993636 12165 consensus_queue.cc:237] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:45.995669 11677 raft_consensus.cc:1261] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:45.995736 11315 raft_consensus.cc:1261] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:45.996218 12164 consensus_queue.cc:1035] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:45.996528 11979 consensus_queue.cc:1035] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:45.999171 12165 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:45.999748 11677 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.000810 11315 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.004391 10776 catalog_manager.cc:5484] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:46.008754 10922 consensus_queue.cc:237] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:46.010416 11677 raft_consensus.cc:1261] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:46.011169 12165 consensus_queue.cc:1035] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:46.013417 11979 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.013995 11677 raft_consensus.cc:2932] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.018457 10774 catalog_manager.cc:5484] T 45996f06656a48fc9f8580abfedd7b52 P 3b5daf0ff6564ffa8d73213596c3915f reported cstate change: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.018822 12165 raft_consensus.cc:1050] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2: attempting to promote NON_VOTER 3b5daf0ff6564ffa8d73213596c3915f to VOTER
I20240501 08:13:46.019351 12165 consensus_queue.cc:237] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:46.020459 10769 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 45996f06656a48fc9f8580abfedd7b52 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:46.022773 11296 tablet_service.cc:1514] Processing DeleteTablet for tablet 45996f06656a48fc9f8580abfedd7b52 with delete_type TABLET_DATA_TOMBSTONED (TS 9e4413f7cb3344058795f529a81bfb99 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44162
I20240501 08:13:46.023154 12235 tablet_replica.cc:330] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: stopping tablet replica
I20240501 08:13:46.023346 12235 raft_consensus.cc:2229] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:46.023679 12235 raft_consensus.cc:2258] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:46.024655 12235 ts_tablet_manager.cc:1875] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:46.025719 11677 raft_consensus.cc:1261] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 LEARNER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.026340 11315 raft_consensus.cc:1261] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.026327 12165 consensus_queue.cc:1035] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.026947 12164 consensus_queue.cc:1035] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.031518 12235 ts_tablet_manager.cc:1888] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:46.031661 12235 log.cc:1196] T 45996f06656a48fc9f8580abfedd7b52 P 9e4413f7cb3344058795f529a81bfb99: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/45996f06656a48fc9f8580abfedd7b52
I20240501 08:13:46.032514 10766 catalog_manager.cc:4830] TS 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109): tablet 45996f06656a48fc9f8580abfedd7b52 (table rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]) successfully deleted
I20240501 08:13:46.034328 12164 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.039394 11315 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.040099 10779 catalog_manager.cc:5484] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:46.043468 11677 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3b5daf0ff6564ffa8d73213596c3915f (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } attrs { replace: true } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.051543 10922 consensus_queue.cc:237] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } }
I20240501 08:13:46.053350 11677 raft_consensus.cc:1261] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:46.053880 12165 consensus_queue.cc:1035] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:46.055927 12164 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.056561 11677 raft_consensus.cc:2932] T 4f90d8a9896845f490e09dd5efe1893e P 3b5daf0ff6564ffa8d73213596c3915f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } } }
I20240501 08:13:46.059715 10769 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4f90d8a9896845f490e09dd5efe1893e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:46.061810 10776 catalog_manager.cc:5484] T 4f90d8a9896845f490e09dd5efe1893e P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 5 to 6, VOTER 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3b5daf0ff6564ffa8d73213596c3915f" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 46677 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:46.065549 11296 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f90d8a9896845f490e09dd5efe1893e with delete_type TABLET_DATA_TOMBSTONED (TS 9e4413f7cb3344058795f529a81bfb99 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44162
I20240501 08:13:46.066201 12235 tablet_replica.cc:330] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: stopping tablet replica
I20240501 08:13:46.066390 12235 raft_consensus.cc:2229] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:46.066726 12235 raft_consensus.cc:2258] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:46.067649 12235 ts_tablet_manager.cc:1875] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:46.071842 12235 ts_tablet_manager.cc:1888] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:46.071957 12235 log.cc:1196] T 4f90d8a9896845f490e09dd5efe1893e P 9e4413f7cb3344058795f529a81bfb99: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/4f90d8a9896845f490e09dd5efe1893e
I20240501 08:13:46.072768 10766 catalog_manager.cc:4830] TS 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109): tablet 4f90d8a9896845f490e09dd5efe1893e (table rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]) successfully deleted
I20240501 08:13:46.098539 12197 raft_consensus.cc:1050] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99: attempting to promote NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 to VOTER
I20240501 08:13:46.099287 12197 consensus_queue.cc:237] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.101995 10922 raft_consensus.cc:1261] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.102043 11808 raft_consensus.cc:1261] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.102627 12196 consensus_queue.cc:1035] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.103026 12002 consensus_queue.cc:1035] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.107429 12002 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.108639 10922 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.112650 10776 catalog_manager.cc:5484] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:46.113574 11808 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.120141 11315 consensus_queue.cc:237] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.121879 11808 raft_consensus.cc:1261] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:46.122367 12196 consensus_queue.cc:1035] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:46.124362 12197 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.127365 10766 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0dda417c185b428bb3456057d9ac2f4c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:46.127787 11808 raft_consensus.cc:2932] T 0dda417c185b428bb3456057d9ac2f4c P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.128682 10779 catalog_manager.cc:5484] T 0dda417c185b428bb3456057d9ac2f4c P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:46.133045 10903 tablet_service.cc:1514] Processing DeleteTablet for tablet 0dda417c185b428bb3456057d9ac2f4c with delete_type TABLET_DATA_TOMBSTONED (TS c56f97cf6c5a4160bd5458de3568e5b2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:48716
I20240501 08:13:46.134331 12280 tablet_replica.cc:330] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: stopping tablet replica
I20240501 08:13:46.134790 12280 raft_consensus.cc:2229] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:46.135219 12280 raft_consensus.cc:2258] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:46.137120 12280 ts_tablet_manager.cc:1875] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:46.143599 12280 ts_tablet_manager.cc:1888] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:46.143747 12280 log.cc:1196] T 0dda417c185b428bb3456057d9ac2f4c P c56f97cf6c5a4160bd5458de3568e5b2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/0dda417c185b428bb3456057d9ac2f4c
I20240501 08:13:46.144779 10769 catalog_manager.cc:4830] TS c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053): tablet 0dda417c185b428bb3456057d9ac2f4c (table rebalance_test_table_0 [id=276389422fdb4636b9ab3946d83ac1a4]) successfully deleted
I20240501 08:13:46.175666 12196 raft_consensus.cc:1050] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99: attempting to promote NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 to VOTER
I20240501 08:13:46.176560 12196 consensus_queue.cc:237] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.178872 10922 raft_consensus.cc:1261] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.178992 11808 raft_consensus.cc:1261] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.179384 12197 consensus_queue.cc:1035] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.179723 12002 consensus_queue.cc:1035] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.182190 12196 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.182762 10922 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.182884 11808 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.186902 10774 catalog_manager.cc:5484] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.191316 11315 consensus_queue.cc:237] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.192827 11808 raft_consensus.cc:1261] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Refusing update from remote peer 9e4413f7cb3344058795f529a81bfb99: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:13:46.193302 12196 consensus_queue.cc:1035] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:46.195434 12164 raft_consensus.cc:1050] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: attempting to promote NON_VOTER 549bd4093e854fc0b8f822b3b96491c9 to VOTER
I20240501 08:13:46.195389 12002 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.196166 12164 consensus_queue.cc:237] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.197772 11808 raft_consensus.cc:2932] T d07456dce1ed4c3385ee13efa386d8e3 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.198298 10766 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d07456dce1ed4c3385ee13efa386d8e3 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:46.198390 11807 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 LEARNER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.198882 11315 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Refusing update from remote peer c56f97cf6c5a4160bd5458de3568e5b2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:13:46.199405 11979 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.199687 12165 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:13:46.200327 10779 catalog_manager.cc:5484] T d07456dce1ed4c3385ee13efa386d8e3 P 9e4413f7cb3344058795f529a81bfb99 reported cstate change: config changed from index 5 to 6, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "9e4413f7cb3344058795f529a81bfb99" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:46.203554 11979 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.206573 10903 tablet_service.cc:1514] Processing DeleteTablet for tablet d07456dce1ed4c3385ee13efa386d8e3 with delete_type TABLET_DATA_TOMBSTONED (TS c56f97cf6c5a4160bd5458de3568e5b2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:48716
I20240501 08:13:46.206977 12280 tablet_replica.cc:330] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: stopping tablet replica
I20240501 08:13:46.207182 12280 raft_consensus.cc:2229] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:46.207506 12280 raft_consensus.cc:2258] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:46.207507 10776 catalog_manager.cc:5484] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 reported cstate change: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:46.207684 11315 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.208385 12280 ts_tablet_manager.cc:1875] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:46.209319 11808 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:46.215049 12280 ts_tablet_manager.cc:1888] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:13:46.215180 12280 log.cc:1196] T d07456dce1ed4c3385ee13efa386d8e3 P c56f97cf6c5a4160bd5458de3568e5b2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/d07456dce1ed4c3385ee13efa386d8e3
I20240501 08:13:46.215915 10769 catalog_manager.cc:4830] TS c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053): tablet d07456dce1ed4c3385ee13efa386d8e3 (table rebalance_test_table_1 [id=743d8b765fc1495a955a459cf84e571a]) successfully deleted
I20240501 08:13:46.395766 10922 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75"
dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38950
I20240501 08:13:46.396044 10922 raft_consensus.cc:595] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:46.425931 10922 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75"
dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38974
I20240501 08:13:46.426118 10922 raft_consensus.cc:595] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:46.643267 10922 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75"
dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38992
I20240501 08:13:46.643462 10922 raft_consensus.cc:595] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Received request to transfer leadership
I20240501 08:13:46.759513 12343 raft_consensus.cc:979] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: : Instructing follower 549bd4093e854fc0b8f822b3b96491c9 to start an election
I20240501 08:13:46.759672 12343 raft_consensus.cc:1067] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Signalling peer 549bd4093e854fc0b8f822b3b96491c9 to start an election
I20240501 08:13:46.760565 11808 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75"
dest_uuid: "549bd4093e854fc0b8f822b3b96491c9"
from {username='slave'} at 127.28.1.129:35501
I20240501 08:13:46.760835 11808 raft_consensus.cc:482] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240501 08:13:46.761019 11808 raft_consensus.cc:3037] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:46.762957 11808 raft_consensus.cc:504] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.764495 11808 leader_election.cc:288] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [CANDIDATE]: Term 2 election: Requested vote from peers 9e4413f7cb3344058795f529a81bfb99 (127.28.1.132:45109), c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053)
I20240501 08:13:46.764964 11315 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75" candidate_uuid: "549bd4093e854fc0b8f822b3b96491c9" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "9e4413f7cb3344058795f529a81bfb99"
I20240501 08:13:46.765007 10922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06d8b52123f641ce9d3b466f48726c75" candidate_uuid: "549bd4093e854fc0b8f822b3b96491c9" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "c56f97cf6c5a4160bd5458de3568e5b2"
I20240501 08:13:46.765190 11315 raft_consensus.cc:3037] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:46.765241 10922 raft_consensus.cc:3032] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Stepping down as leader of term 1
I20240501 08:13:46.765324 10922 raft_consensus.cc:729] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: c56f97cf6c5a4160bd5458de3568e5b2, State: Running, Role: LEADER
I20240501 08:13:46.765579 10922 consensus_queue.cc:260] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.765898 10922 raft_consensus.cc:3037] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 1 FOLLOWER]: Advancing to term 2
I20240501 08:13:46.767067 11315 raft_consensus.cc:2454] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 549bd4093e854fc0b8f822b3b96491c9 in term 2.
I20240501 08:13:46.767649 11741 leader_election.cc:302] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 549bd4093e854fc0b8f822b3b96491c9, 9e4413f7cb3344058795f529a81bfb99; no voters:
I20240501 08:13:46.767755 10922 raft_consensus.cc:2454] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 549bd4093e854fc0b8f822b3b96491c9 in term 2.
I20240501 08:13:46.768299 12344 raft_consensus.cc:2781] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 2 FOLLOWER]: Leader election won for term 2
I20240501 08:13:46.768671 12344 raft_consensus.cc:686] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 2 LEADER]: Becoming Leader. State: Replica: 549bd4093e854fc0b8f822b3b96491c9, State: Running, Role: LEADER
I20240501 08:13:46.769165 12344 consensus_queue.cc:237] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:46.772480 10776 catalog_manager.cc:5484] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 reported cstate change: term changed from 1 to 2, leader changed from c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) to 549bd4093e854fc0b8f822b3b96491c9 (127.28.1.131). New cstate: current_term: 2 leader_uuid: "549bd4093e854fc0b8f822b3b96491c9" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:47.177160 10922 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Refusing update from remote peer 549bd4093e854fc0b8f822b3b96491c9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:13:47.177809 12344 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c56f97cf6c5a4160bd5458de3568e5b2" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 33053 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:47.181931 11315 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Refusing update from remote peer 549bd4093e854fc0b8f822b3b96491c9: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240501 08:13:47.182562 12344 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:13:47.631006 11808 consensus_queue.cc:237] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } }
I20240501 08:13:47.632743 11315 raft_consensus.cc:1261] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Refusing update from remote peer 549bd4093e854fc0b8f822b3b96491c9: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240501 08:13:47.633322 12373 consensus_queue.cc:1035] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:13:47.635437 12345 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:47.636090 11315 raft_consensus.cc:2932] T 06d8b52123f641ce9d3b466f48726c75 P 9e4413f7cb3344058795f529a81bfb99 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } } }
I20240501 08:13:47.638347 10766 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 06d8b52123f641ce9d3b466f48726c75 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:13:47.639742 10779 catalog_manager.cc:5484] T 06d8b52123f641ce9d3b466f48726c75 P 549bd4093e854fc0b8f822b3b96491c9 reported cstate change: config changed from index 5 to 7, VOTER c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129) evicted. New cstate: current_term: 2 leader_uuid: "549bd4093e854fc0b8f822b3b96491c9" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "9e4413f7cb3344058795f529a81bfb99" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 45109 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "549bd4093e854fc0b8f822b3b96491c9" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 34007 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:13:47.644035 10903 tablet_service.cc:1514] Processing DeleteTablet for tablet 06d8b52123f641ce9d3b466f48726c75 with delete_type TABLET_DATA_TOMBSTONED (TS c56f97cf6c5a4160bd5458de3568e5b2 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:48716
I20240501 08:13:47.644706 12378 tablet_replica.cc:330] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: stopping tablet replica
I20240501 08:13:47.644965 12378 raft_consensus.cc:2229] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240501 08:13:47.645244 12378 raft_consensus.cc:2258] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240501 08:13:47.646298 12378 ts_tablet_manager.cc:1875] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:13:47.650494 12378 ts_tablet_manager.cc:1888] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240501 08:13:47.650600 12378 log.cc:1196] T 06d8b52123f641ce9d3b466f48726c75 P c56f97cf6c5a4160bd5458de3568e5b2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/06d8b52123f641ce9d3b466f48726c75
I20240501 08:13:47.651448 10769 catalog_manager.cc:4830] TS c56f97cf6c5a4160bd5458de3568e5b2 (127.28.1.129:33053): tablet 06d8b52123f641ce9d3b466f48726c75 (table rebalance_test_table_2 [id=c0289a7b65f2496a852d2c8fa7134ca4]) successfully deleted
I20240501 08:13:47.777149 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:47.781543 11788 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:47.790792 10903 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:47.803373 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:47.981712 10903 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:47.982286 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:47.985426 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:47.992717 11788 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:48.114204 11657 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:13:48.121729 10903 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:48.122129 11788 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240501 08:13:48.126967 11296 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:13:48.307076 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10841
I20240501 08:13:48.320484 10963 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:48.447383 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10841
W20240501 08:13:48.466945 11612 connection.cc:535] server connection from 127.28.1.129:37197 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:13:48.467537 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 11595
I20240501 08:13:48.473169 11717 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:48.575151 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 11595
I20240501 08:13:48.591485 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 11726
I20240501 08:13:48.597376 11847 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:48.703918 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 11726
W20240501 08:13:48.720074 10768 connection.cc:535] server connection from 127.28.1.131:32961 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240501 08:13:48.720139 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 11234
I20240501 08:13:48.726084 11356 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:48.848743 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 11234
I20240501 08:13:48.867386 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 10751
I20240501 08:13:48.868606 10812 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:48.999497 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 10751
2024-05-01T08:13:49Z chronyd exiting
[ OK ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1 (19597 ms)
[ RUN ] LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer/idx6_rf7_t7_l_3_4_3
I20240501 08:13:49.045539 28678 test_util.cc:268] Using random seed: 284482312
I20240501 08:13:49.047427 28678 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:13:49.047499 28678 ts_itest-base.cc:113] --------------
I20240501 08:13:49.047539 28678 ts_itest-base.cc:114] 10 tablet servers
I20240501 08:13:49.047578 28678 ts_itest-base.cc:115] 7 replicas per TS
I20240501 08:13:49.047622 28678 ts_itest-base.cc:116] --------------
2024-05-01T08:13:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:13:49Z Disabled control of system clock
I20240501 08:13:49.093379 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:37815
--webserver_interface=127.28.1.190
--webserver_port=0
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.28.1.190:37815
--location_mapping_cmd=/tmp/dist-test-taskVWMLaH/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240501 08:13:49.420429 12500 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240501 08:13:49.420790 12500 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:49.420888 12500 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:49.420989 12500 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:49.428354 12500 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:49.428512 12500 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240501 08:13:49.428614 12500 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:49.428674 12500 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240501 08:13:49.428761 12500 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240501 08:13:49.437947 12500 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskVWMLaH/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--ipki_ca_key_size=768
--master_addresses=127.28.1.190:37815
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.28.1.190:37815
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.28.1.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:49.439308 12500 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:49.441053 12500 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:49.452631 12508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:49.453102 12505 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:49.453531 12506 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:49.454100 12500 server_base.cc:1034] running on GCE node
I20240501 08:13:49.454895 12500 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:49.456396 12500 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:49.457625 12500 hybrid_clock.cc:648] HybridClock initialized: now 1714551229457559 us; error 46 us; skew 500 ppm
I20240501 08:13:49.457947 12500 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:49.460196 12500 webserver.cc:460] Webserver started at http://127.28.1.190:35591/ using document root <none> and password file <none>
I20240501 08:13:49.460928 12500 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:49.461056 12500 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:49.461344 12500 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:49.463922 12500 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "11335367f1a64c02ab69b90dbd1b5334"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.464751 12500 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "11335367f1a64c02ab69b90dbd1b5334"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.469791 12500 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:49.473177 12514 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:49.474473 12500 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:49.474637 12500 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
uuid: "11335367f1a64c02ab69b90dbd1b5334"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.474803 12500 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:49.503103 12500 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:49.504122 12500 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:49.504455 12500 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:49.523082 12500 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.190:37815
I20240501 08:13:49.523133 12565 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.190:37815 every 8 connection(s)
I20240501 08:13:49.524487 12500 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240501 08:13:49.528317 12566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:49.530987 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 12500
I20240501 08:13:49.531219 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:13:49.539602 12566 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334: Bootstrap starting.
I20240501 08:13:49.542703 12566 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:49.544121 12566 log.cc:824] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:49.546922 12566 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334: No bootstrap required, opened a new log
I20240501 08:13:49.553103 12566 raft_consensus.cc:348] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } }
I20240501 08:13:49.553444 12566 raft_consensus.cc:374] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:49.553522 12566 raft_consensus.cc:729] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11335367f1a64c02ab69b90dbd1b5334, State: Initialized, Role: FOLLOWER
I20240501 08:13:49.554055 12566 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } }
I20240501 08:13:49.554241 12566 raft_consensus.cc:388] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240501 08:13:49.554338 12566 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240501 08:13:49.554452 12566 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:49.556790 12566 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } }
I20240501 08:13:49.557236 12566 leader_election.cc:302] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 11335367f1a64c02ab69b90dbd1b5334; no voters:
I20240501 08:13:49.557770 12566 leader_election.cc:288] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240501 08:13:49.557897 12571 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:49.558454 12571 raft_consensus.cc:686] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [term 1 LEADER]: Becoming Leader. State: Replica: 11335367f1a64c02ab69b90dbd1b5334, State: Running, Role: LEADER
I20240501 08:13:49.558977 12571 consensus_queue.cc:237] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } }
I20240501 08:13:49.559648 12566 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [sys.catalog]: configured and running, proceeding with master startup.
I20240501 08:13:49.561558 12572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "11335367f1a64c02ab69b90dbd1b5334" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } } }
I20240501 08:13:49.561858 12572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:49.562637 12573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11335367f1a64c02ab69b90dbd1b5334. Latest consensus state: current_term: 1 leader_uuid: "11335367f1a64c02ab69b90dbd1b5334" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "11335367f1a64c02ab69b90dbd1b5334" member_type: VOTER last_known_addr { host: "127.28.1.190" port: 37815 } } }
I20240501 08:13:49.562887 12573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334 [sys.catalog]: This master's current role is: LEADER
I20240501 08:13:49.564734 12576 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240501 08:13:49.570070 12576 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240501 08:13:49.579286 12576 catalog_manager.cc:1347] Generated new cluster ID: 3e808c5a523e4ce2829a81cb1b61eda0
I20240501 08:13:49.579411 12576 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240501 08:13:49.586766 12576 catalog_manager.cc:1370] Generated new certificate authority record
I20240501 08:13:49.588016 12576 catalog_manager.cc:1504] Loading token signing keys...
I20240501 08:13:49.597720 12576 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 11335367f1a64c02ab69b90dbd1b5334: Generated new TSK 0
I20240501 08:13:49.598644 12576 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240501 08:13:49.605024 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.129:0
--local_ip_for_outbound_sockets=127.28.1.129
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:13:49.941619 12591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:49.941920 12591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:49.942015 12591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:49.949590 12591 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:49.949721 12591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:49.949846 12591 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.129
I20240501 08:13:49.958900 12591 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.28.1.129
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.129
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:49.960245 12591 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:49.961934 12591 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:49.973474 12596 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:49.974057 12597 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:49.974479 12599 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:49.975054 12591 server_base.cc:1034] running on GCE node
I20240501 08:13:49.975602 12591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:49.976560 12591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:49.977757 12591 hybrid_clock.cc:648] HybridClock initialized: now 1714551229977695 us; error 54 us; skew 500 ppm
I20240501 08:13:49.978065 12591 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:49.980244 12591 webserver.cc:460] Webserver started at http://127.28.1.129:45791/ using document root <none> and password file <none>
I20240501 08:13:49.981009 12591 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:49.981117 12591 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:49.981417 12591 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:49.984004 12591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "56e1ccecd51040458c27911fa34a90a6"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.984766 12591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "56e1ccecd51040458c27911fa34a90a6"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.989789 12591 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240501 08:13:49.993125 12605 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:49.994412 12591 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
I20240501 08:13:49.994572 12591 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "56e1ccecd51040458c27911fa34a90a6"
format_stamp: "Formatted at 2024-05-01 08:13:49 on dist-test-slave-jj4n"
I20240501 08:13:49.994756 12591 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:50.007746 12591 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:50.008570 12591 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:50.008797 12591 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:50.009645 12591 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:50.011016 12591 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:50.011101 12591 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.011179 12591 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:50.011232 12591 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.039918 12591 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.129:45349
I20240501 08:13:50.039995 12718 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.129:45349 every 8 connection(s)
I20240501 08:13:50.041239 12591 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240501 08:13:50.047750 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 12591
I20240501 08:13:50.047984 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:13:50.051640 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.130:0
--local_ip_for_outbound_sockets=127.28.1.130
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:50.053850 12719 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:50.054124 12719 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:50.054884 12719 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:50.098294 12531 ts_manager.cc:194] Registered new tserver with Master: 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:13:50.100311 12531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.129:48767
W20240501 08:13:50.381860 12723 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:50.382202 12723 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:50.382344 12723 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:50.389905 12723 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:50.390059 12723 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:50.390211 12723 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.130
I20240501 08:13:50.399478 12723 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.28.1.130
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.130
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:50.400848 12723 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:50.402523 12723 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:50.414068 12729 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:50.414381 12730 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:50.414768 12732 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:50.415319 12723 server_base.cc:1034] running on GCE node
I20240501 08:13:50.415925 12723 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:50.416921 12723 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:50.418138 12723 hybrid_clock.cc:648] HybridClock initialized: now 1714551230418075 us; error 59 us; skew 500 ppm
I20240501 08:13:50.418463 12723 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:50.420696 12723 webserver.cc:460] Webserver started at http://127.28.1.130:38489/ using document root <none> and password file <none>
I20240501 08:13:50.421435 12723 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:50.421559 12723 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:50.421859 12723 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:50.426102 12723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.426931 12723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.432210 12723 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240501 08:13:50.435572 12738 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.436957 12723 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:50.437117 12723 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.437314 12723 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:50.467381 12723 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:50.468315 12723 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:50.468585 12723 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:50.469565 12723 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:50.471062 12723 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:50.471158 12723 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.471287 12723 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:50.471357 12723 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.500875 12723 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.130:45135
I20240501 08:13:50.500943 12851 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.130:45135 every 8 connection(s)
I20240501 08:13:50.502148 12723 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240501 08:13:50.508963 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 12723
I20240501 08:13:50.509159 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:13:50.512702 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.131:0
--local_ip_for_outbound_sockets=127.28.1.131
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:50.515457 12852 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:50.515810 12852 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:50.516835 12852 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:50.575402 12531 ts_manager.cc:194] Registered new tserver with Master: 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135)
I20240501 08:13:50.576349 12531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.130:46441
W20240501 08:13:50.858945 12855 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:50.859247 12855 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:50.859378 12855 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:50.866901 12855 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:50.867029 12855 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:50.867151 12855 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.131
I20240501 08:13:50.876233 12855 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.28.1.131
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.131
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:50.877519 12855 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:50.879221 12855 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:50.891402 12862 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:50.891705 12861 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:50.892318 12864 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:50.893054 12855 server_base.cc:1034] running on GCE node
I20240501 08:13:50.893625 12855 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:50.894620 12855 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:50.895823 12855 hybrid_clock.cc:648] HybridClock initialized: now 1714551230895772 us; error 43 us; skew 500 ppm
I20240501 08:13:50.896179 12855 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:50.898422 12855 webserver.cc:460] Webserver started at http://127.28.1.131:34059/ using document root <none> and password file <none>
I20240501 08:13:50.899219 12855 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:50.899327 12855 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:50.899647 12855 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:50.902331 12855 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "5bf85222f4734b7885eb866efd27d5f2"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.903087 12855 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "5bf85222f4734b7885eb866efd27d5f2"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.908367 12855 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:13:50.911924 12870 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.913426 12855 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:50.913604 12855 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "5bf85222f4734b7885eb866efd27d5f2"
format_stamp: "Formatted at 2024-05-01 08:13:50 on dist-test-slave-jj4n"
I20240501 08:13:50.913808 12855 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:50.925817 12855 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:50.926622 12855 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:50.926873 12855 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:50.927778 12855 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:50.929283 12855 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:50.929380 12855 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.929494 12855 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:50.929569 12855 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:50.959614 12855 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.131:38689
I20240501 08:13:50.959690 12983 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.131:38689 every 8 connection(s)
I20240501 08:13:50.960987 12855 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240501 08:13:50.966008 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 12855
I20240501 08:13:50.966239 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:13:50.969714 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.132:0
--local_ip_for_outbound_sockets=127.28.1.132
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:50.974685 12984 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:50.974985 12984 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:50.975822 12984 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:51.034562 12531 ts_manager.cc:194] Registered new tserver with Master: 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:13:51.035501 12531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.131:50095
I20240501 08:13:51.103770 12719 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:51.325251 12987 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:51.325577 12987 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:51.325718 12987 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:51.333159 12987 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:51.333319 12987 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:51.333478 12987 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.132
I20240501 08:13:51.342660 12987 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.28.1.132
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.132
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:51.344022 12987 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:51.345857 12987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:51.357960 12996 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:51.358402 12993 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:51.359117 12994 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:51.359627 12987 server_base.cc:1034] running on GCE node
I20240501 08:13:51.360313 12987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:51.361356 12987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:51.362591 12987 hybrid_clock.cc:648] HybridClock initialized: now 1714551231362532 us; error 49 us; skew 500 ppm
I20240501 08:13:51.362931 12987 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:51.365231 12987 webserver.cc:460] Webserver started at http://127.28.1.132:42109/ using document root <none> and password file <none>
I20240501 08:13:51.366003 12987 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:51.366113 12987 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:51.366426 12987 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:51.369330 12987 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.370093 12987 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.376101 12987 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
I20240501 08:13:51.379528 13002 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.381007 12987 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:13:51.381172 12987 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.381408 12987 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:51.402426 12987 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:51.403277 12987 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:51.403548 12987 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:51.404556 12987 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:51.406049 12987 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:51.406150 12987 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.406261 12987 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:51.406329 12987 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.435617 12987 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.132:43293
I20240501 08:13:51.435688 13115 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.132:43293 every 8 connection(s)
I20240501 08:13:51.436920 12987 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240501 08:13:51.446127 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 12987
I20240501 08:13:51.446296 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:13:51.449393 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.133:0
--local_ip_for_outbound_sockets=127.28.1.133
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:51.450270 13116 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:51.450575 13116 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:51.451539 13116 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:51.501051 12531 ts_manager.cc:194] Registered new tserver with Master: f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:13:51.502003 12531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.132:45333
I20240501 08:13:51.578976 12852 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:51.793506 13119 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:51.793810 13119 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:51.793918 13119 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:51.801440 13119 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:51.801575 13119 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:51.801698 13119 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.133
I20240501 08:13:51.810840 13119 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.28.1.133
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.133
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:51.812160 13119 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:51.813817 13119 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:51.825395 13125 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:51.825809 13126 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:51.826427 13129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:51.827173 13119 server_base.cc:1034] running on GCE node
I20240501 08:13:51.827802 13119 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:51.828778 13119 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:51.829967 13119 hybrid_clock.cc:648] HybridClock initialized: now 1714551231829902 us; error 60 us; skew 500 ppm
I20240501 08:13:51.830269 13119 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:51.832562 13119 webserver.cc:460] Webserver started at http://127.28.1.133:36743/ using document root <none> and password file <none>
I20240501 08:13:51.833297 13119 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:51.833410 13119 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:51.833706 13119 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:51.836357 13119 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.837116 13119 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.842301 13119 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240501 08:13:51.845820 13135 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.847262 13119 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:51.847426 13119 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
format_stamp: "Formatted at 2024-05-01 08:13:51 on dist-test-slave-jj4n"
I20240501 08:13:51.847622 13119 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:51.862061 13119 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:51.862978 13119 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:51.863240 13119 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:51.864265 13119 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:51.865844 13119 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:51.865931 13119 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.866014 13119 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:51.866066 13119 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:51.895771 13119 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.133:42529
I20240501 08:13:51.895817 13248 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.133:42529 every 8 connection(s)
I20240501 08:13:51.897156 13119 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240501 08:13:51.903800 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13119
I20240501 08:13:51.903995 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240501 08:13:51.907552 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.134:0
--local_ip_for_outbound_sockets=127.28.1.134
--webserver_interface=127.28.1.134
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:51.910594 13249 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:51.910908 13249 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:51.911773 13249 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:51.954031 12531 ts_manager.cc:194] Registered new tserver with Master: fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:13:51.954938 12531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.133:46371
I20240501 08:13:52.038087 12984 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:52.238850 13252 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:52.239187 13252 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:52.239320 13252 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:52.247148 13252 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:52.247303 13252 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:52.247455 13252 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.134
I20240501 08:13:52.256759 13252 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.28.1.134
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.134
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:52.258065 13252 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:52.259752 13252 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:52.271828 13258 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:52.272166 13261 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:52.272564 13259 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:52.273356 13252 server_base.cc:1034] running on GCE node
I20240501 08:13:52.274017 13252 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:52.274988 13252 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:52.276171 13252 hybrid_clock.cc:648] HybridClock initialized: now 1714551232276111 us; error 53 us; skew 500 ppm
I20240501 08:13:52.276507 13252 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:52.278718 13252 webserver.cc:460] Webserver started at http://127.28.1.134:40057/ using document root <none> and password file <none>
I20240501 08:13:52.279484 13252 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:52.279604 13252 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:52.279919 13252 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:52.282651 13252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.283424 13252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.288674 13252 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:13:52.292132 13267 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.293490 13252 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:52.293643 13252 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.293838 13252 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:52.310637 13252 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:52.311496 13252 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:52.311729 13252 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:52.312697 13252 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:52.314128 13252 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:52.314215 13252 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.314323 13252 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:52.314393 13252 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.343561 13252 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.134:37699
I20240501 08:13:52.343632 13379 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.134:37699 every 8 connection(s)
I20240501 08:13:52.344930 13252 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240501 08:13:52.355154 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13252
I20240501 08:13:52.355345 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240501 08:13:52.358258 13380 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:52.358570 13380 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:52.358821 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.135:0
--local_ip_for_outbound_sockets=127.28.1.135
--webserver_interface=127.28.1.135
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:52.359476 13380 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:52.402913 12530 ts_manager.cc:194] Registered new tserver with Master: c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:13:52.403934 12530 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.134:50593
I20240501 08:13:52.504644 13116 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:52.696457 13385 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:52.696766 13385 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:52.696908 13385 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:52.704538 13385 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:52.704679 13385 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:52.704828 13385 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.135
I20240501 08:13:52.713832 13385 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.28.1.135
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.135
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:52.715169 13385 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:52.716929 13385 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:52.728490 13390 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:52.728992 13393 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:52.729593 13391 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:52.730157 13385 server_base.cc:1034] running on GCE node
I20240501 08:13:52.730746 13385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:52.731696 13385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:52.732905 13385 hybrid_clock.cc:648] HybridClock initialized: now 1714551232732852 us; error 46 us; skew 500 ppm
I20240501 08:13:52.733225 13385 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:52.735359 13385 webserver.cc:460] Webserver started at http://127.28.1.135:34317/ using document root <none> and password file <none>
I20240501 08:13:52.736119 13385 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:52.736230 13385 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:52.736533 13385 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:52.739066 13385 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.739825 13385 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.745116 13385 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240501 08:13:52.748576 13399 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.749955 13385 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:52.750121 13385 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
format_stamp: "Formatted at 2024-05-01 08:13:52 on dist-test-slave-jj4n"
I20240501 08:13:52.750315 13385 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:52.773854 13385 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:52.774739 13385 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:52.775010 13385 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:52.776033 13385 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:52.777647 13385 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:52.777745 13385 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.777853 13385 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:52.777920 13385 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:52.810245 13385 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.135:43031
I20240501 08:13:52.810297 13512 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.135:43031 every 8 connection(s)
I20240501 08:13:52.811594 13385 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240501 08:13:52.817466 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13385
I20240501 08:13:52.817704 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240501 08:13:52.821028 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.136:0
--local_ip_for_outbound_sockets=127.28.1.136
--webserver_interface=127.28.1.136
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:52.824520 13513 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:52.824820 13513 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:52.825650 13513 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:52.898468 12530 ts_manager.cc:194] Registered new tserver with Master: b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:13:52.899682 12530 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.135:46785
I20240501 08:13:52.957662 13249 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:53.159149 13516 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:53.159487 13516 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:53.159632 13516 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:53.167200 13516 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:53.167349 13516 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:53.167500 13516 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.136
I20240501 08:13:53.176631 13516 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.28.1.136
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.136
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:53.177964 13516 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:53.179697 13516 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:53.192147 13525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:53.192651 13522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:53.193120 13523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:53.193626 13516 server_base.cc:1034] running on GCE node
I20240501 08:13:53.194243 13516 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:53.195206 13516 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:53.196405 13516 hybrid_clock.cc:648] HybridClock initialized: now 1714551233196351 us; error 43 us; skew 500 ppm
I20240501 08:13:53.196744 13516 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:53.202893 13516 webserver.cc:460] Webserver started at http://127.28.1.136:42427/ using document root <none> and password file <none>
I20240501 08:13:53.203680 13516 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:53.203792 13516 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:53.204162 13516 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:53.206848 13516 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "35cb9fadf231408598c3877837a6149e"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.207633 13516 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "35cb9fadf231408598c3877837a6149e"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.212980 13516 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240501 08:13:53.216569 13531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.218043 13516 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
I20240501 08:13:53.218215 13516 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "35cb9fadf231408598c3877837a6149e"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.218416 13516 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:53.236604 13516 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:53.237514 13516 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:53.237790 13516 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:53.238777 13516 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:53.240435 13516 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:53.240528 13516 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.240610 13516 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:53.240664 13516 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.270885 13516 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.136:37957
I20240501 08:13:53.270956 13644 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.136:37957 every 8 connection(s)
I20240501 08:13:53.272255 13516 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240501 08:13:53.279873 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13516
I20240501 08:13:53.280067 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240501 08:13:53.283475 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.137:0
--local_ip_for_outbound_sockets=127.28.1.137
--webserver_interface=127.28.1.137
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:53.285118 13645 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:53.285421 13645 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:53.286267 13645 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:53.344624 12530 ts_manager.cc:194] Registered new tserver with Master: 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957)
I20240501 08:13:53.345985 12530 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.136:44869
I20240501 08:13:53.406606 13380 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:53.638765 13648 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:53.639073 13648 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:53.639168 13648 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:53.646718 13648 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:53.646848 13648 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:53.646975 13648 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.137
I20240501 08:13:53.656062 13648 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.137:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.28.1.137
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.137
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:53.657303 13648 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:53.658926 13648 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:53.671168 13654 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:53.671698 13655 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:53.672057 13657 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:53.673841 13648 server_base.cc:1034] running on GCE node
I20240501 08:13:53.674535 13648 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:53.675595 13648 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:53.676792 13648 hybrid_clock.cc:648] HybridClock initialized: now 1714551233676741 us; error 50 us; skew 500 ppm
I20240501 08:13:53.677114 13648 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:53.679304 13648 webserver.cc:460] Webserver started at http://127.28.1.137:37645/ using document root <none> and password file <none>
I20240501 08:13:53.680124 13648 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:53.680241 13648 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:53.680562 13648 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:53.683130 13648 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "8e309f48ce254660915fb9fc32649ab2"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.683857 13648 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "8e309f48ce254660915fb9fc32649ab2"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.689173 13648 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.000s
I20240501 08:13:53.692783 13663 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.694190 13648 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240501 08:13:53.694360 13648 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "8e309f48ce254660915fb9fc32649ab2"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:13:53.694566 13648 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:53.710975 13648 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:53.711850 13648 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:53.712144 13648 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:53.713094 13648 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:53.714582 13648 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:53.714676 13648 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.714781 13648 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:53.714848 13648 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:53.744563 13648 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.137:37989
I20240501 08:13:53.744594 13776 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.137:37989 every 8 connection(s)
I20240501 08:13:53.745939 13648 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240501 08:13:53.746459 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13648
I20240501 08:13:53.746681 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240501 08:13:53.750635 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.138:0
--local_ip_for_outbound_sockets=127.28.1.138
--webserver_interface=127.28.1.138
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:13:53.759781 13777 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:53.760244 13777 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:53.761344 13777 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:53.805248 12530 ts_manager.cc:194] Registered new tserver with Master: 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989)
I20240501 08:13:53.806202 12530 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.137:60853
I20240501 08:13:53.902266 13513 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
W20240501 08:13:54.090381 13780 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:54.090714 13780 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:13:54.090839 13780 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:54.098426 13780 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:13:54.098585 13780 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:13:54.098766 13780 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.138
I20240501 08:13:54.108155 13780 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.138:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.28.1.138
--webserver_port=0
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.138
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:13:54.109472 13780 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:13:54.111302 13780 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:13:54.123162 13786 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:54.123409 13787 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:13:54.124055 13789 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:13:54.124708 13780 server_base.cc:1034] running on GCE node
I20240501 08:13:54.125299 13780 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:13:54.126238 13780 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:13:54.127454 13780 hybrid_clock.cc:648] HybridClock initialized: now 1714551234127378 us; error 70 us; skew 500 ppm
I20240501 08:13:54.127787 13780 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:13:54.129997 13780 webserver.cc:460] Webserver started at http://127.28.1.138:38787/ using document root <none> and password file <none>
I20240501 08:13:54.130878 13780 fs_manager.cc:359] Metadata directory not provided
I20240501 08:13:54.131006 13780 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:13:54.131361 13780 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240501 08:13:54.134652 13780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/instance:
uuid: "3cd5ec97de8642dd8796fae63ea73e5a"
format_stamp: "Formatted at 2024-05-01 08:13:54 on dist-test-slave-jj4n"
I20240501 08:13:54.135625 13780 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal/instance:
uuid: "3cd5ec97de8642dd8796fae63ea73e5a"
format_stamp: "Formatted at 2024-05-01 08:13:54 on dist-test-slave-jj4n"
I20240501 08:13:54.140810 13780 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
I20240501 08:13:54.144109 13795 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:54.145538 13780 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240501 08:13:54.145679 13780 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "3cd5ec97de8642dd8796fae63ea73e5a"
format_stamp: "Formatted at 2024-05-01 08:13:54 on dist-test-slave-jj4n"
I20240501 08:13:54.145856 13780 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:13:54.163641 13780 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:13:54.164520 13780 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:13:54.164775 13780 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:13:54.165728 13780 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:13:54.167225 13780 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:13:54.167308 13780 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:54.167412 13780 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:13:54.167475 13780 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:13:54.195402 13780 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.138:40307
I20240501 08:13:54.195454 13907 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.138:40307 every 8 connection(s)
I20240501 08:13:54.196691 13780 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240501 08:13:54.202996 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 13780
I20240501 08:13:54.203183 28678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal/instance
I20240501 08:13:54.209899 13908 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:13:54.210192 13908 heartbeater.cc:464] Registering TS with master...
I20240501 08:13:54.210984 13908 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:13:54.254153 12530 ts_manager.cc:194] Registered new tserver with Master: 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307)
I20240501 08:13:54.255350 12530 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.138:42573
I20240501 08:13:54.258335 28678 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240501 08:13:54.276084 28678 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240501 08:13:54.276247 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 13648
I20240501 08:13:54.283818 13772 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:54.349447 13645 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
I20240501 08:13:54.364207 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 13648
I20240501 08:13:54.376948 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 13780
I20240501 08:13:54.386108 13903 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:54.467381 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 13780
I20240501 08:13:54.479699 28678 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu pid 13516
I20240501 08:13:54.487730 13640 generic_service.cc:196] Checking for leaks (request via RPC)
I20240501 08:13:54.573565 28678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu with pid 13516
I20240501 08:13:58.338821 12530 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:13:58.345629 12530 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:13:58.401121 13048 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.402482 13050 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.402730 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.406520 12652 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.406064 12917 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.407400 12651 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.407523 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.407516 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.407955 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.403021 13049 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.408120 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.409207 12916 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.409686 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.410635 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.411255 12918 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.411733 12918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.406054 12653 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.428007 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.439813 13944 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:13:58.445423 13943 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:13:58.451439 13944 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.453125 13944 log.cc:824] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.453264 12786 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.454856 12786 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.455345 13943 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.456225 12785 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.456732 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.456693 13943 log.cc:824] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.462262 13942 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:13:58.465998 13942 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.466820 13313 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.467334 13943 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:13:58.467344 13942 log.cc:824] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.467672 13943 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.023s user 0.009s sys 0.004s
I20240501 08:13:58.468293 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.472990 13315 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.473577 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.473827 13183 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.474723 13944 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:13:58.475062 13944 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.036s user 0.009s sys 0.005s
I20240501 08:13:58.475303 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.476461 13181 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.476586 12784 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.476922 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.477145 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.463577 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.480768 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.484612 13944 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.485299 13944 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.485487 13944 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.485186 13445 tablet_service.cc:1467] Processing CreateTablet for tablet 8a12727f56884a45bcd89631b986eab7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.486478 13944 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.486734 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.488485 13182 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.489068 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.490219 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 795dd1b66844485dad769acd4ec2b16f (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.490752 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.496172 13447 tablet_service.cc:1467] Processing CreateTablet for tablet 3c85628288804001a4777a0f00269836 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:13:58.496340 13944 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.021s user 0.017s sys 0.000s
I20240501 08:13:58.496709 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:13:58.496940 13944 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:13:58.498565 13944 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.507072 13944 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:13:58.507334 13944 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20240501 08:13:58.508397 13944 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.508759 13944 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.508900 13944 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.509301 13944 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.510061 13944 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:13:58.510406 13944 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:13:58.512178 13944 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.521497 13948 raft_consensus.cc:482] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.521764 13948 raft_consensus.cc:504] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.524603 13948 leader_election.cc:288] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:13:58.532567 13952 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:13:58.537755 13953 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:13:58.538784 13944 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:13:58.540375 13944 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.030s user 0.002s sys 0.008s
I20240501 08:13:58.542424 13952 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.543035 13944 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.543864 13952 log.cc:824] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.544358 13944 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.544811 13944 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.545091 13953 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.545493 13944 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.546355 13953 log.cc:824] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.547397 13944 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:13:58.565105 13953 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:13:58.565627 13953 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.028s user 0.011s sys 0.005s
I20240501 08:13:58.569305 13952 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:13:58.569666 13952 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.037s user 0.008s sys 0.007s
I20240501 08:13:58.571557 13959 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:13:58.574223 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.575414 13959 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
W20240501 08:13:58.575718 13004 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.576995 13960 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:13:58.580945 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.583775 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
W20240501 08:13:58.585062 13005 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:13:58.585616 13005 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.587383 13952 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.587918 13952 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.588045 13952 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.585825 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.589484 13953 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
W20240501 08:13:58.589871 13006 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.589982 13953 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.590082 13953 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.590057 13006 leader_election.cc:302] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: f59d8b4fd79049b6b40dcecd13d0be9d; no voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:13:58.590560 13948 raft_consensus.cc:2726] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:13:58.590965 13952 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.592088 13953 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.594941 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.593559 13943 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
W20240501 08:13:58.596047 13005 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.597014 13960 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.597240 13943 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.597402 13943 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.597684 13959 log.cc:824] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.598698 13942 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:13:58.595889 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:13:58.599023 13942 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.137s user 0.044s sys 0.026s
I20240501 08:13:58.598980 13943 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
W20240501 08:13:58.599980 13006 leader_election.cc:341] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.601054 13960 log.cc:824] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Log is configured to *not* fsync() on all Append() calls
I20240501 08:13:58.604935 13943 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.137s user 0.005s sys 0.009s
I20240501 08:13:58.605422 13943 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:13:58.605571 13942 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.606065 13942 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.606164 13942 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.606961 13942 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.607419 13943 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.608189 13953 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.042s user 0.013s sys 0.003s
I20240501 08:13:58.609933 13953 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:13:58.611610 13953 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.613788 13952 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.044s user 0.016s sys 0.001s
I20240501 08:13:58.614246 13952 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:13:58.615912 13952 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.616511 13943 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:13:58.617018 13943 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.000s
I20240501 08:13:58.619522 13942 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.020s user 0.009s sys 0.006s
I20240501 08:13:58.619724 13959 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:13:58.619942 13942 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:13:58.620039 13959 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.049s user 0.006s sys 0.008s
I20240501 08:13:58.621654 13942 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.623792 13953 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:13:58.623996 13953 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.014s user 0.006s sys 0.001s
I20240501 08:13:58.625065 13953 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.625398 13953 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.625501 13953 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.625509 13960 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:13:58.625877 13960 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.049s user 0.014s sys 0.000s
I20240501 08:13:58.626037 13959 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.626502 13959 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.626611 13959 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.626717 13952 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:13:58.626897 13952 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.003s
I20240501 08:13:58.627467 13959 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.627910 13952 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.628221 13952 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.628327 13952 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.628657 13952 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.629249 13952 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:58.629611 13952 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:13:58.628757 13943 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.629979 13943 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.630844 13953 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.631248 13943 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.631628 13953 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240501 08:13:58.631911 13953 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:13:58.631633 13943 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.633584 13953 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.633848 13943 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.017s user 0.002s sys 0.000s
I20240501 08:13:58.634969 13948 raft_consensus.cc:482] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.635161 13952 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.635160 13948 raft_consensus.cc:504] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.635524 13943 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:13:58.636418 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.636831 13948 leader_election.cc:288] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:13:58.637382 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.637615 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:13:58.637737 12673 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:13:58.637895 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.638049 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:13:58.638228 13203 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
W20240501 08:13:58.638696 13005 leader_election.cc:341] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:13:58.639235 13006 leader_election.cc:341] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:13:58.639472 13005 leader_election.cc:341] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.639923 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.640224 13943 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.640295 12938 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:13:58.640760 13006 leader_election.cc:302] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 7 responses out of 7 voters: 4 yes votes; 3 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29
I20240501 08:13:58.641381 13948 raft_consensus.cc:2781] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:58.641523 13948 raft_consensus.cc:482] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:58.641613 13948 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.642153 13960 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.643035 13953 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:13:58.643051 13960 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.643225 13953 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.000s
I20240501 08:13:58.643244 13960 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.643996 13953 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.644304 13953 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.644381 13953 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.644523 13960 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.644639 13953 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.644711 13948 raft_consensus.cc:504] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.646174 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:13:58.646371 12806 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.647785 13953 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240501 08:13:58.648447 13952 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:13:58.648629 13952 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.001s
I20240501 08:13:58.649070 12806 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:13:58.650403 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:13:58.650642 12938 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.650833 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:13:58.650858 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:13:58.651068 12673 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
W20240501 08:13:58.651355 13005 leader_election.cc:341] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.652416 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:13:58.652558 13952 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.652832 13952 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.652905 13952 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
W20240501 08:13:58.652948 13006 leader_election.cc:341] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:13:58.653090 13948 leader_election.cc:288] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:13:58.653193 13952 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.653656 13942 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:13:58.653807 13942 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.034s user 0.008s sys 0.004s
I20240501 08:13:58.653908 12673 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:13:58.654564 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:13:58.654796 13203 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.654793 13942 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.655185 13942 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.655300 13942 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.655611 13942 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.656203 13942 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:13:58.656508 13942 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:13:58.657550 13203 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:13:58.654011 13952 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.005s user 0.002s sys 0.001s
I20240501 08:13:58.657932 13005 leader_election.cc:302] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29
I20240501 08:13:58.658205 12938 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:13:58.658366 13948 raft_consensus.cc:2781] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:58.659766 13959 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.040s user 0.013s sys 0.001s
I20240501 08:13:58.660485 13959 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:13:58.660862 13960 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.035s user 0.012s sys 0.000s
I20240501 08:13:58.661271 13960 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:13:58.662176 13959 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.663933 13943 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:13:58.664160 13943 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.029s user 0.003s sys 0.004s
I20240501 08:13:58.664542 13942 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.665016 13943 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.665455 13943 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.665616 13943 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.665974 13943 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.666822 13943 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240501 08:13:58.667798 13960 tablet_bootstrap.cc:654] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.670630 13959 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:13:58.670816 13959 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.006s
I20240501 08:13:58.671872 13959 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.672199 13959 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.672297 13959 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.672602 13959 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.673336 13959 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:13:58.673629 13959 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:13:58.675348 13959 tablet_bootstrap.cc:654] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.676398 13948 raft_consensus.cc:686] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Becoming Leader. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Running, Role: LEADER
I20240501 08:13:58.677071 13948 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.677628 13960 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:13:58.677807 13960 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.017s user 0.002s sys 0.008s
I20240501 08:13:58.677976 13959 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:13:58.678136 13959 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240501 08:13:58.678493 13942 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:13:58.678687 13942 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.022s user 0.007s sys 0.000s
I20240501 08:13:58.678772 13960 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.678805 13959 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.679055 13960 raft_consensus.cc:374] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.679073 13959 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.679142 13959 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.679144 13960 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.679380 13959 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.679425 13942 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.679828 13942 raft_consensus.cc:374] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.679366 13960 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.679919 13942 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.680524 13960 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.003s user 0.001s sys 0.002s
I20240501 08:13:58.680531 13942 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.681175 13942 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:13:58.681242 13960 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:13:58.682113 13959 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:13:58.683398 13960 tablet_bootstrap.cc:654] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:13:58.685638 12531 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: term changed from 0 to 1, leader changed from <none> to f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132). New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:58.687425 13960 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:13:58.687604 13960 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.004s
I20240501 08:13:58.688382 13960 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.688706 13960 raft_consensus.cc:374] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:13:58.688782 13960 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:13:58.689029 13960 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.689616 13960 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
W20240501 08:13:58.693503 13117 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240501 08:13:58.722540 12985 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:58.737632 13977 raft_consensus.cc:482] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.737905 13977 raft_consensus.cc:504] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.740234 13977 leader_election.cc:288] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:13:58.750787 13964 raft_consensus.cc:482] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.751036 13964 raft_consensus.cc:504] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.753954 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:13:58.754379 13335 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.755550 13964 leader_election.cc:288] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:13:58.763620 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.763919 12938 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
W20240501 08:13:58.759790 12853 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:58.771409 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.771721 13203 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.772269 13402 leader_election.cc:302] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:13:58.772770 13977 raft_consensus.cc:2781] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:58.772934 13977 raft_consensus.cc:482] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:58.773020 13977 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.774772 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.775035 12673 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.776283 13977 raft_consensus.cc:504] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.778126 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:13:58.778344 12938 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.778641 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:13:58.778852 13335 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1be03ffeb3384b42bfee6ffaf3f451c9 in term 0.
I20240501 08:13:58.780282 13977 leader_election.cc:288] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Requested vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:13:58.780699 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:13:58.780900 13203 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.781219 12938 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:13:58.782701 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:13:58.782902 12673 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.783618 13203 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:13:58.784075 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:13:58.784283 13335 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.785521 12673 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:13:58.790632 13402 leader_election.cc:302] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:13:58.791460 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.792244 13335 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:13:58.792372 12806 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.792822 13977 raft_consensus.cc:2781] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:58.793979 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:13:58.794144 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:13:58.794386 12806 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.794422 13070 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.794919 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:13:58.795182 13069 raft_consensus.cc:3037] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.797570 12806 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:13:58.797956 13069 raft_consensus.cc:2454] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
W20240501 08:13:58.798923 12720 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:58.800760 13996 raft_consensus.cc:482] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.800931 13996 raft_consensus.cc:504] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.802589 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.802930 12806 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate b17fe4a1f54c4655b85eb663fcff1b93 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.804446 13963 raft_consensus.cc:482] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.804646 13963 raft_consensus.cc:504] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:13:58.805213 13977 raft_consensus.cc:686] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Becoming Leader. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Running, Role: LEADER
I20240501 08:13:58.805841 13977 consensus_queue.cc:237] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:13:58.807502 13963 leader_election.cc:288] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:13:58.810091 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:13:58.810345 13335 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:13:58.811548 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.811903 13203 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate b17fe4a1f54c4655b85eb663fcff1b93 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.812637 13996 leader_election.cc:288] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
W20240501 08:13:58.817330 13514 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:58.832702 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:13:58.833091 13069 raft_consensus.cc:2379] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 1be03ffeb3384b42bfee6ffaf3f451c9 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:13:58.833814 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.834167 12673 raft_consensus.cc:2379] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 1be03ffeb3384b42bfee6ffaf3f451c9 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:13:58.834276 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.834570 12938 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate b17fe4a1f54c4655b85eb663fcff1b93 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.838310 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:13:58.839845 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.840191 13203 raft_consensus.cc:2379] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 1be03ffeb3384b42bfee6ffaf3f451c9 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:13:58.840279 12530 catalog_manager.cc:5484] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: term changed from 0 to 1, leader changed from <none> to b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135). New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } }
I20240501 08:13:58.841097 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:13:58.842113 12672 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.842478 12672 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c58c4517fd6941f19a9e75d1e48e8b29 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.842769 12742 leader_election.cc:302] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, c58c4517fd6941f19a9e75d1e48e8b29; no voters: 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:13:58.842937 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c85628288804001a4777a0f00269836" candidate_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.843232 12938 raft_consensus.cc:2379] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 1be03ffeb3384b42bfee6ffaf3f451c9 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:13:58.843262 13964 raft_consensus.cc:2726] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:13:58.843904 13401 leader_election.cc:302] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:13:58.844457 13998 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.845062 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.845317 12673 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate b17fe4a1f54c4655b85eb663fcff1b93 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.847615 13998 raft_consensus.cc:2726] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240501 08:13:58.850760 13381 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:58.852270 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:13:58.852569 13467 raft_consensus.cc:2454] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.852975 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.853237 12806 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c58c4517fd6941f19a9e75d1e48e8b29 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.855424 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.855708 12938 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c58c4517fd6941f19a9e75d1e48e8b29 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.857044 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.857318 13203 raft_consensus.cc:2379] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c58c4517fd6941f19a9e75d1e48e8b29 in current term 1: Already voted for candidate f59d8b4fd79049b6b40dcecd13d0be9d in this term.
I20240501 08:13:58.857256 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "795dd1b66844485dad769acd4ec2b16f" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:13:58.857734 13270 leader_election.cc:302] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:13:58.858125 13963 raft_consensus.cc:3037] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.860086 13963 raft_consensus.cc:2726] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:13:58.871728 13963 raft_consensus.cc:482] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:13:58.872020 13963 raft_consensus.cc:504] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.873378 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:13:58.873374 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:13:58.873512 13963 leader_election.cc:288] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:13:58.873638 13203 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.873637 13069 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.874029 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:13:58.874104 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:13:58.874264 12806 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.874213 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:13:58.874369 12938 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.874456 12673 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.874874 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:13:58.875094 13467 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:13:58.875075 13269 leader_election.cc:302] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:13:58.875391 13963 raft_consensus.cc:2781] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:13:58.875504 13963 raft_consensus.cc:482] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:13:58.875613 13963 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.877522 13963 raft_consensus.cc:504] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.878510 13963 leader_election.cc:288] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Requested vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:13:58.878624 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:13:58.878625 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:13:58.878857 13069 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.878885 13203 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.878841 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:13:58.879032 12673 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.881054 13069 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.881673 12673 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.881669 13203 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.881839 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:13:58.881878 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:13:58.882045 13467 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.882115 12938 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.882212 13270 leader_election.cc:302] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:13:58.882498 13963 raft_consensus.cc:2781] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:13:58.882642 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8a12727f56884a45bcd89631b986eab7" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:13:58.882840 12806 raft_consensus.cc:3037] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:13:58.884814 12938 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.884827 13467 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.885740 13963 raft_consensus.cc:686] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Becoming Leader. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Running, Role: LEADER
I20240501 08:13:58.885828 12806 raft_consensus.cc:2454] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:13:58.886374 13963 consensus_queue.cc:237] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:13:58.891518 12530 catalog_manager.cc:5484] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: term changed from 0 to 1, leader changed from <none> to c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134). New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
W20240501 08:13:58.904116 13250 tablet.cc:2369] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240501 08:13:59.089845 13948 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.138159 13948 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.193977 13948 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.194358 14013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.216114 13998 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.220047 13998 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.232270 14025 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.232977 14025 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.261548 13998 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.268111 13996 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.269644 13996 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.282804 13998 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.304410 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:13:59.353519 14010 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:13:59.353875 14010 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:13:59.362176 14010 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:13:59.367573 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.385798 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.391368 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.392098 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:13:59.400288 13963 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 452.871 ms
time per row: 0.226436 ms
I20240501 08:13:59.991709 12527 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:13:59.992431 12527 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:00.001590 12786 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.001752 12653 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.002066 12786 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.002209 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.002518 13182 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.002939 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.005555 12917 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.005692 13447 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.006009 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.006103 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.006227 12652 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.006615 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.006683 12918 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.007046 12918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.008590 12785 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.008977 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.010143 13181 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.010535 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.011698 12651 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.012110 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.012573 12784 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.012946 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.016417 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.016844 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.017030 12916 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.017463 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.020519 13183 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.020942 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.021493 13445 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.021883 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.024677 13315 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.025120 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.025091 13313 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.025462 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.025389 13048 tablet_service.cc:1467] Processing CreateTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.025796 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.028383 13050 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.028788 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.030745 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 85eb7c44993b4749b4b0a090d773bbef (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.031139 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.032344 13049 tablet_service.cc:1467] Processing CreateTablet for tablet 7ac049d157cd4cc495c747c02be17bca (DEFAULT_TABLE table=rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:00.032719 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:00.037719 14124 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:00.039611 14124 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.040107 14125 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:00.042241 14125 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.048591 14125 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:00.049430 14125 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240501 08:14:00.050554 14125 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.052630 14125 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.052939 14125 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.053527 14125 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.055199 14125 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:14:00.056089 14125 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:00.060721 14125 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.072525 14128 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:00.074457 14128 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.074760 14130 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:00.076442 14131 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:00.079769 14131 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.082252 14129 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:00.084669 14130 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.086336 14123 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:00.086465 14129 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.088465 14124 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:00.088536 14131 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:00.088657 14124 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.051s user 0.026s sys 0.021s
I20240501 08:14:00.088708 14131 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240501 08:14:00.089576 14124 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.089584 14131 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.089888 14131 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.089892 14124 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.089967 14131 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.089975 14124 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.090206 14124 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.091008 14124 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:14:00.091327 14124 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:00.091953 14130 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:00.092147 14130 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.018s user 0.004s sys 0.000s
I20240501 08:14:00.093022 14124 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.093024 14130 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.094576 14130 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.094678 14130 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.095067 14125 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:00.095000 14131 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.095376 14125 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.039s user 0.003s sys 0.018s
I20240501 08:14:00.095754 14123 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.096259 14125 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.096514 14131 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.008s user 0.003s sys 0.000s
I20240501 08:14:00.096671 14125 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.096799 14125 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.096859 14131 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:00.097159 14125 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.098009 14125 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:00.098564 14125 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:00.095919 14130 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.099589 14128 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:00.099758 14128 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.027s user 0.004s sys 0.000s
I20240501 08:14:00.100347 14125 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.100625 14128 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.100916 14128 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.101011 14128 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.101280 14128 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.102375 14124 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:00.102614 14124 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.003s
I20240501 08:14:00.103628 14124 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.104022 14124 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.104147 14124 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.104874 14124 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.105592 14124 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:00.106048 14124 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:00.107767 14131 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.109481 14124 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.110774 14128 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.011s user 0.005s sys 0.000s
I20240501 08:14:00.113202 14125 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:00.113442 14125 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.002s
I20240501 08:14:00.114261 14128 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:00.115938 14128 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.115748 14125 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.116178 14125 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.116319 14125 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.116662 14125 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.117472 14125 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:14:00.120362 14131 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:00.120537 14131 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.024s user 0.003s sys 0.000s
I20240501 08:14:00.121157 14131 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.121645 14131 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.121737 14131 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.122005 14131 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.122509 14130 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.030s user 0.004s sys 0.000s
I20240501 08:14:00.122869 14130 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:00.123387 14124 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:00.123582 14124 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.001s
I20240501 08:14:00.124466 14130 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.124449 14124 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.124765 14124 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.124861 14124 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.125604 14124 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.125900 14128 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:00.126060 14128 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240501 08:14:00.126777 14124 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240501 08:14:00.127108 14141 raft_consensus.cc:482] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.127281 14141 raft_consensus.cc:504] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.130882 14141 leader_election.cc:288] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:00.131754 14128 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.132184 14128 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.132323 14128 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.132607 14128 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.133339 14128 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:00.134253 14128 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:00.135915 14128 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.149797 14131 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.029s user 0.002s sys 0.000s
I20240501 08:14:00.150188 14131 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:00.154119 14129 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:00.154310 14129 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.072s user 0.012s sys 0.012s
I20240501 08:14:00.155112 14123 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:00.155318 14123 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.069s user 0.013s sys 0.013s
I20240501 08:14:00.155123 14129 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.155437 14129 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.155539 14129 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.156306 14034 raft_consensus.cc:482] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.156493 14034 raft_consensus.cc:504] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.157841 14034 leader_election.cc:288] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:00.158625 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:00.159056 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.159283 13335 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
W20240501 08:14:00.159322 13403 leader_election.cc:341] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.159627 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:00.159897 13202 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.160989 14131 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.161548 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.161958 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
W20240501 08:14:00.162256 13401 leader_election.cc:341] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.162577 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.162245 14123 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.162756 14123 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.162808 13069 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.162851 14123 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.163815 14123 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.164216 12805 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.165030 13401 leader_election.cc:302] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2
I20240501 08:14:00.165401 14034 raft_consensus.cc:2781] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:00.165516 14034 raft_consensus.cc:482] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:00.165616 14034 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.166394 14130 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:00.166424 14131 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:00.166651 14130 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.044s user 0.004s sys 0.000s
I20240501 08:14:00.166684 14131 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.000s
I20240501 08:14:00.167455 14130 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.167477 14131 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.167744 14130 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.167819 14130 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.167852 14131 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.168059 14131 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.168116 14130 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.168512 14034 raft_consensus.cc:504] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.168841 14130 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:00.168427 14131 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.169767 14131 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:00.169762 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:00.169976 12805 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.171288 14130 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:00.172750 12805 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.174850 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:00.175025 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:00.175237 13202 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
W20240501 08:14:00.175424 13403 leader_election.cc:341] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.176425 14130 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.177719 14123 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.022s user 0.002s sys 0.000s
I20240501 08:14:00.178102 14123 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:00.178135 13202 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.179278 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:00.179504 13335 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.179805 14123 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.182216 13335 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.182674 13402 leader_election.cc:302] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 5bf85222f4734b7885eb866efd27d5f2
I20240501 08:14:00.183002 13977 raft_consensus.cc:2781] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:00.183154 13977 raft_consensus.cc:686] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Becoming Leader. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Running, Role: LEADER
I20240501 08:14:00.183499 13977 consensus_queue.cc:237] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.187990 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:00.188364 14034 leader_election.cc:288] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
W20240501 08:14:00.188733 13138 leader_election.cc:341] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.188866 14130 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:00.189042 14130 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.020s user 0.002s sys 0.003s
I20240501 08:14:00.189935 14130 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.190245 14130 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.190315 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.190568 12671 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:00.191677 14123 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:00.191792 14034 raft_consensus.cc:482] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.191893 14123 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.014s user 0.005s sys 0.000s
I20240501 08:14:00.191946 14034 raft_consensus.cc:504] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.193256 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.193426 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:00.193410 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.193495 13069 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:00.193629 13335 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:00.190340 14130 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.194044 13138 leader_election.cc:302] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9
I20240501 08:14:00.194036 14123 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.194439 14123 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.194517 14123 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.194510 14141 raft_consensus.cc:2781] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:00.194345 14130 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.194622 14141 raft_consensus.cc:482] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:00.194727 14141 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.194933 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:00.195060 14130 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:14:00.195171 14123 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.195492 12805 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.195828 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:00.196048 14123 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:14:00.196382 14123 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
W20240501 08:14:00.196413 13403 leader_election.cc:341] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.197181 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.197492 13335 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.198319 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:00.197511 14141 raft_consensus.cc:504] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.198522 12671 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.198572 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
W20240501 08:14:00.199137 13402 leader_election.cc:341] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.199559 14141 leader_election.cc:288] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:00.200090 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:00.200299 13334 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.200398 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:00.200629 13069 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.201195 12671 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.201930 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:00.202104 12671 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.202929 13334 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:00.203464 13069 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:00.203626 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:00.204602 12671 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:00.205519 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:00.205592 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.205718 13070 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.205848 12671 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.206008 13137 leader_election.cc:302] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 5bf85222f4734b7885eb866efd27d5f2
I20240501 08:14:00.206656 13401 leader_election.cc:302] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29; no voters: 5bf85222f4734b7885eb866efd27d5f2, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:00.206996 13998 raft_consensus.cc:2781] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:00.207098 13998 raft_consensus.cc:482] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:00.207202 13998 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.208276 13070 raft_consensus.cc:2454] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.208555 14141 raft_consensus.cc:2781] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:00.209434 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.209676 13070 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:00.209578 14129 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.209955 12936 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:00.209967 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:00.209897 13998 raft_consensus.cc:504] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.211321 14123 tablet_bootstrap.cc:654] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
W20240501 08:14:00.211480 13138 leader_election.cc:341] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.212523 14163 raft_consensus.cc:482] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.212766 14163 raft_consensus.cc:504] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.213498 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
W20240501 08:14:00.214274 13403 leader_election.cc:341] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.215011 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:00.215255 12805 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.216859 14128 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:00.217047 14128 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.083s user 0.024s sys 0.029s
I20240501 08:14:00.219729 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:00.219996 12671 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.220412 14128 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.220773 14128 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.220876 14128 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.221417 13998 leader_election.cc:288] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:00.221904 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:00.222142 13070 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.222499 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:00.222561 12671 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.222507 12528 catalog_manager.cc:5484] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: term changed from 0 to 1, leader changed from <none> to b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135). New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:00.223553 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:00.223006 12805 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.224694 13070 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.225466 13401 leader_election.cc:302] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d; no voters: 5bf85222f4734b7885eb866efd27d5f2
I20240501 08:14:00.226421 14128 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.226987 13202 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.227404 13998 raft_consensus.cc:2781] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:00.227680 13998 raft_consensus.cc:686] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Becoming Leader. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Running, Role: LEADER
I20240501 08:14:00.228046 13998 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.229138 13335 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.234133 13202 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.236727 12528 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: term changed from 0 to 1, leader changed from <none> to b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135). New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:00.234822 14141 raft_consensus.cc:686] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Becoming Leader. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Running, Role: LEADER
I20240501 08:14:00.238207 14141 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.242381 14128 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.025s user 0.009s sys 0.001s
I20240501 08:14:00.245702 14034 leader_election.cc:288] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:00.247861 13335 raft_consensus.cc:2454] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:00.259225 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.259544 12671 raft_consensus.cc:2379] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate fa5b68f497b84b31ac35e5549b2af1ad in this term.
I20240501 08:14:00.268442 14129 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.114s user 0.015s sys 0.032s
I20240501 08:14:00.268882 14129 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:00.270854 14129 tablet_bootstrap.cc:654] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.272310 14163 leader_election.cc:288] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:00.276461 14129 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:00.276690 14129 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.001s
I20240501 08:14:00.277164 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.277336 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:00.277451 13070 raft_consensus.cc:2379] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate fa5b68f497b84b31ac35e5549b2af1ad in this term.
I20240501 08:14:00.277501 14129 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.277791 14129 raft_consensus.cc:374] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.277886 14129 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
W20240501 08:14:00.278025 12873 leader_election.cc:341] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:00.278147 14129 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.278750 14129 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:00.279119 14129 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:00.279578 14123 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:00.280865 14129 tablet_bootstrap.cc:654] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:00.283062 14129 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:00.283057 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.283221 14129 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240501 08:14:00.283380 13335 raft_consensus.cc:2379] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate fa5b68f497b84b31ac35e5549b2af1ad in this term.
I20240501 08:14:00.284020 12873 leader_election.cc:302] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:00.284026 14129 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.284310 14129 raft_consensus.cc:374] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.283854 12527 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: term changed from 0 to 1, leader changed from <none> to fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133). New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:00.285357 14123 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.089s user 0.009s sys 0.000s
I20240501 08:14:00.286372 14129 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.286226 14123 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.286486 14123 raft_consensus.cc:374] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:00.286592 14123 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:00.286630 14129 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.286968 14163 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.286860 14123 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.289007 14129 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.006s user 0.000s sys 0.003s
I20240501 08:14:00.289757 14163 raft_consensus.cc:2726] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:00.290921 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:00.291015 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:00.291193 13467 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:00.292874 14123 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.007s user 0.001s sys 0.001s
I20240501 08:14:00.293666 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:00.293915 13466 raft_consensus.cc:2454] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:00.297449 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:00.418776 14163 raft_consensus.cc:482] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.419086 14163 raft_consensus.cc:504] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.420403 14163 leader_election.cc:288] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:00.420712 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:00.420701 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.420979 13070 raft_consensus.cc:2379] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.421003 12805 raft_consensus.cc:2379] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.421540 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.421574 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.421813 13335 raft_consensus.cc:2379] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.421834 12671 raft_consensus.cc:2379] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.421742 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:00.422406 12874 leader_election.cc:302] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:00.422609 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ac049d157cd4cc495c747c02be17bca" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:00.422734 14163 raft_consensus.cc:3037] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.422863 13202 raft_consensus.cc:2379] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.425441 14163 raft_consensus.cc:2726] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:00.480096 14163 raft_consensus.cc:482] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:00.480286 14163 raft_consensus.cc:504] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:00.481666 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:00.482020 12805 raft_consensus.cc:2379] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.482170 14163 leader_election.cc:288] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:00.482208 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:00.482460 13335 raft_consensus.cc:2379] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.482693 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:00.482944 13202 raft_consensus.cc:2379] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.483207 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:00.483410 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:00.483644 13070 raft_consensus.cc:2379] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.483803 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85eb7c44993b4749b4b0a090d773bbef" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:00.484072 12671 raft_consensus.cc:2379] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5bf85222f4734b7885eb866efd27d5f2 in current term 1: Already voted for candidate b17fe4a1f54c4655b85eb663fcff1b93 in this term.
I20240501 08:14:00.484036 12874 leader_election.cc:302] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:00.484334 14163 raft_consensus.cc:3037] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.486271 14163 raft_consensus.cc:2726] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:00.591506 14034 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.591797 13998 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.650875 12805 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.651638 14034 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.654265 14141 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.658720 14141 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.663131 13466 raft_consensus.cc:3037] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:00.665884 14141 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.680523 14164 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.682375 14141 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.685971 13998 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:00.685395 14179 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:00.686385 14179 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:14:00.688189 14164 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.698083 13998 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:00.700251 14179 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:14:00.716905 13977 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.717373 13977 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.717612 14201 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.725935 14034 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.728165 14034 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.734629 14034 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:00.750845 14210 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_1'
INSERT report
rows total: 2000
time total: 401.824 ms
time per row: 0.200912 ms
I20240501 08:14:01.310042 12527 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:14:01.310789 12527 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:01.319753 13182 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.320252 13048 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.320741 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.321151 13315 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.321576 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.321607 12652 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.322011 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.324503 12784 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.324939 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.325438 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.328009 12651 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.328413 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.328855 12786 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.329223 12786 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.332365 12785 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.332798 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.333638 13445 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.333717 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.334101 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.334340 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.334786 13447 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.335196 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.336783 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.337222 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.340452 13313 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.340919 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.340876 12917 tablet_service.cc:1467] Processing CreateTablet for tablet 9f65863a31714bd19691d8eb5187ca91 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.341322 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.342015 13050 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.342443 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.344544 12918 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.344794 13049 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.344820 13183 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.344983 12918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.345198 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.345198 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.348415 13181 tablet_service.cc:1467] Processing CreateTablet for tablet 2b098b5bce924e6ea5897e463451b964 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.348415 12916 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.348706 12653 tablet_service.cc:1467] Processing CreateTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (DEFAULT_TABLE table=rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:01.348841 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.348838 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.349128 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:01.367280 14286 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:01.370318 14286 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.378695 14288 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:01.382684 14288 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.387732 14288 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:01.387916 14288 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.000s
I20240501 08:14:01.388537 14283 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:01.390106 14287 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:01.391942 14288 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.392386 14288 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.393056 14283 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.387749 14286 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:01.393221 14288 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.393436 14286 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.004s
I20240501 08:14:01.393860 14287 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.393697 14288 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.394265 14286 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.394919 14285 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:01.396682 14285 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.394557 14286 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.397603 14286 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.397912 14286 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.398133 14284 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:01.398605 14286 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.005s user 0.000s sys 0.001s
I20240501 08:14:01.399196 14286 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:01.407768 14283 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:01.408064 14283 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.022s user 0.009s sys 0.000s
I20240501 08:14:01.408545 14286 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.409271 14283 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.409933 14283 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.410096 14283 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.410161 14284 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.411171 14285 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:01.411402 14285 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.004s
I20240501 08:14:01.412297 14287 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:01.412555 14283 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.412297 14285 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.413466 14285 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.413959 14287 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.024s user 0.004s sys 0.000s
I20240501 08:14:01.414786 14283 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:01.414741 14287 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.415036 14287 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.415124 14287 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.414945 14285 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.415431 14287 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.416182 14283 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:01.417115 14282 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:01.415585 14285 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.418987 14282 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.420367 14288 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.029s user 0.000s sys 0.009s
I20240501 08:14:01.420719 14288 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:01.420914 14287 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:01.421494 14287 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:01.423274 14287 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.423523 14284 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:01.423717 14284 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.026s user 0.003s sys 0.000s
I20240501 08:14:01.425930 14288 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.427482 14285 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.016s user 0.000s sys 0.002s
I20240501 08:14:01.428027 14284 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.428352 14284 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.428442 14284 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.428763 14284 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.429543 14284 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.006s user 0.000s sys 0.003s
I20240501 08:14:01.429884 14284 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:01.431643 14284 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.432929 13977 raft_consensus.cc:482] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:01.433105 13977 raft_consensus.cc:504] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.434363 13977 leader_election.cc:288] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:01.434572 14291 raft_consensus.cc:482] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:01.434867 14285 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:01.434835 14291 raft_consensus.cc:504] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.436245 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:01.436542 14285 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.436875 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:01.437891 14283 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.439651 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:01.439940 14286 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:01.440187 14286 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.041s user 0.007s sys 0.014s
W20240501 08:14:01.440420 13403 leader_election.cc:341] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.435863 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:01.440975 14286 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.441293 14286 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20240501 08:14:01.441295 13402 leader_election.cc:341] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.441373 14286 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
W20240501 08:14:01.441522 13402 leader_election.cc:341] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:14:01.441715 13402 leader_election.cc:341] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.441623 14286 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.441875 13402 leader_election.cc:302] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:01.441921 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:01.442127 12673 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:01.442529 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
W20240501 08:14:01.443298 13401 leader_election.cc:341] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.443544 14298 raft_consensus.cc:482] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:01.443714 14298 raft_consensus.cc:504] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.445281 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
W20240501 08:14:01.445945 12872 leader_election.cc:341] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.446146 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:01.446430 13203 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.446830 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:01.446990 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
W20240501 08:14:01.447518 12872 leader_election.cc:341] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:14:01.448031 12874 leader_election.cc:341] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.448583 13977 raft_consensus.cc:2726] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:01.448534 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:01.448768 12805 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.449395 14298 leader_election.cc:288] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135)
I20240501 08:14:01.456039 14291 leader_election.cc:288] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:01.468534 14288 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:01.468847 14288 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.048s user 0.001s sys 0.004s
I20240501 08:14:01.469787 14288 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.470214 14288 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.470341 14288 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.470649 14288 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.472724 14282 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:01.472959 14282 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.056s user 0.004s sys 0.003s
I20240501 08:14:01.474018 14282 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.474406 14282 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.474534 14282 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.474845 14282 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.476521 14282 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:01.476892 14282 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:01.483239 14283 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:01.483764 14288 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.015s user 0.002s sys 0.000s
I20240501 08:14:01.484071 14283 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.068s user 0.001s sys 0.005s
I20240501 08:14:01.485849 14288 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:01.487018 14283 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.487413 14283 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.442220 14286 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:01.447664 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:01.488890 14283 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.489368 14286 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:01.490273 14288 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.489259 14283 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.490846 14286 tablet_bootstrap.cc:654] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.491581 14283 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:01.492713 14287 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:01.492961 14287 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.072s user 0.004s sys 0.005s
W20240501 08:14:01.493635 12873 leader_election.cc:341] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:01.493796 12873 leader_election.cc:302] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:01.494097 14287 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.494364 14298 raft_consensus.cc:2726] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:01.494496 14287 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.494618 14287 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.494942 14287 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.495618 14287 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:01.495859 14285 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:01.496053 14285 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.061s user 0.004s sys 0.000s
I20240501 08:14:01.496239 14287 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:01.496801 14285 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.497083 14285 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.497165 14285 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.497977 14283 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:01.499626 14287 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.500653 14285 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.501605 14282 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.503239 14283 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.517177 14285 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.021s user 0.002s sys 0.000s
I20240501 08:14:01.517665 14285 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:01.518466 14283 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:01.519580 14287 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:01.521085 14286 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:01.521265 14286 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.032s user 0.000s sys 0.002s
I20240501 08:14:01.522009 14282 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:01.522130 14287 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.025s user 0.000s sys 0.004s
I20240501 08:14:01.522078 14286 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.522352 14286 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.522444 14286 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.522751 14283 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.025s user 0.004s sys 0.001s
I20240501 08:14:01.522718 14286 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.523470 14286 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240501 08:14:01.523456 14282 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.046s user 0.006s sys 0.000s
I20240501 08:14:01.524310 14283 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.525346 14282 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.525780 14285 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.526222 14287 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.526651 14287 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.526785 14283 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.526928 14287 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.527065 14282 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.527186 14283 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.527487 14287 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.527805 14282 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.528270 14282 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.528692 14283 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.529376 14287 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240501 08:14:01.529852 14283 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:01.531714 14282 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:01.532405 14282 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:01.534143 14288 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:01.534387 14288 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.049s user 0.004s sys 0.002s
I20240501 08:14:01.535341 14288 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.535701 14288 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.536006 14288 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.536432 14288 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.539361 14288 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240501 08:14:01.543510 14282 tablet_bootstrap.cc:654] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.550545 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:01.550859 12805 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.552863 14282 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:01.553112 14282 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.021s user 0.009s sys 0.000s
I20240501 08:14:01.554026 14282 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.554447 14282 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.554594 14282 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.554904 14282 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.555514 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:01.555738 13466 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.556257 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:01.556484 13069 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.556617 14282 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:01.556942 12607 leader_election.cc:302] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:01.557402 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:01.557483 14291 raft_consensus.cc:2781] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:01.557588 14291 raft_consensus.cc:482] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:01.557672 14291 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.557641 13203 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.560813 14291 raft_consensus.cc:504] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.562500 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:01.562762 13069 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.564422 14291 leader_election.cc:288] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:01.565219 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:01.565340 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:01.565456 13203 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.565577 12805 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.568225 13203 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.569855 13069 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.570950 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:01.571183 13466 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.571429 12805 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.571995 12608 leader_election.cc:302] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:01.572381 14291 raft_consensus.cc:2781] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:01.574052 13466 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.575688 14291 raft_consensus.cc:686] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Becoming Leader. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Running, Role: LEADER
I20240501 08:14:01.576459 14291 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.581912 14285 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:01.583349 12527 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:01.584697 14285 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.067s user 0.008s sys 0.008s
I20240501 08:14:01.585681 14285 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.586019 14285 raft_consensus.cc:374] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.586138 14285 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.587424 14285 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.589991 14285 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240501 08:14:01.590862 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:01.591133 12937 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.591442 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:01.591622 12936 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.594324 12936 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.594545 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:01.594770 13333 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:01.595073 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ede6a5ca6654aceb2a24e1592fc1bc8" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:01.595260 13335 raft_consensus.cc:3037] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.597920 14284 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:01.597947 13335 raft_consensus.cc:2454] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:01.598130 14284 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.168s user 0.028s sys 0.051s
I20240501 08:14:01.598917 14284 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.599195 14284 raft_consensus.cc:374] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.599278 14284 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.599529 14284 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.600098 14284 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:14:01.600464 14284 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:01.602133 14284 tablet_bootstrap.cc:654] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:01.607352 14284 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:01.607520 14284 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.004s
I20240501 08:14:01.608165 14284 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.608383 14284 raft_consensus.cc:374] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:01.608479 14284 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:01.608703 14284 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.609264 14284 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:14:01.650117 14298 raft_consensus.cc:482] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:01.650339 14298 raft_consensus.cc:504] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.651510 14298 leader_election.cc:288] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:01.651860 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:01.651890 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:01.652118 12805 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.652122 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:01.652159 13068 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.652333 12673 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.652747 12873 leader_election.cc:302] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:01.653030 14298 raft_consensus.cc:2781] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:01.653162 14298 raft_consensus.cc:482] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:01.653122 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:01.653246 14298 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.653167 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:01.653348 13203 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.653364 13466 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.653914 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:01.654147 13335 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:01.655652 14298 raft_consensus.cc:504] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.656767 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:01.656893 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:01.656899 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:01.656993 12805 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.657083 13203 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.657084 14298 leader_election.cc:288] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:01.657114 12673 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.658591 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:01.658882 13335 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.659664 13203 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.659690 12673 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.660410 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:01.660626 13466 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.661477 13335 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.661918 12873 leader_election.cc:302] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:01.661950 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b098b5bce924e6ea5897e463451b964" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:01.662154 13068 raft_consensus.cc:3037] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.662308 14298 raft_consensus.cc:2781] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:01.662967 13466 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.663434 14298 raft_consensus.cc:686] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Becoming Leader. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Running, Role: LEADER
I20240501 08:14:01.664325 13068 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.664158 14298 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:01.665060 12805 raft_consensus.cc:2454] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:01.668699 12529 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:01.842788 14013 raft_consensus.cc:482] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:01.843006 14013 raft_consensus.cc:504] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.844130 14013 leader_election.cc:288] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:01.844445 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:01.844566 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:01.844681 13335 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.844663 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:01.844760 12673 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.844770 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:01.844895 13466 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.844986 12936 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.845395 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:01.845448 13006 leader_election.cc:302] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:01.845646 13203 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.845750 14013 raft_consensus.cc:2781] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:01.845710 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:01.845893 14013 raft_consensus.cc:482] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:01.845938 12805 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:01.846032 14013 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.848016 14013 raft_consensus.cc:504] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.849134 13335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:01.849207 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:01.849324 13335 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.849435 13466 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.849706 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:01.849725 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:01.849885 12673 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.849920 12936 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.851392 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:01.851598 13203 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.852123 13466 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.852206 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f65863a31714bd19691d8eb5187ca91" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:01.852422 12805 raft_consensus.cc:3037] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:01.852450 12936 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.852978 14013 leader_election.cc:288] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Requested vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:01.851956 13335 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.853610 13005 leader_election.cc:302] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:01.853607 12673 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.854161 14013 raft_consensus.cc:2781] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:01.855043 12805 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.855165 13203 raft_consensus.cc:2454] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:01.855549 14013 raft_consensus.cc:686] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Becoming Leader. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Running, Role: LEADER
I20240501 08:14:01.855859 14013 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:01.858995 12529 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: term changed from 0 to 1, leader changed from <none> to f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132). New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:01.993258 14291 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:01.997140 14325 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.008559 14325 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.012899 14325 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.025418 14325 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.028103 14325 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240501 08:14:02.053256 14298 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.123122 14298 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.152271 14298 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.168251 14350 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.173738 14298 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.177577 14328 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.257102 14013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.304739 14013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.338011 14013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:02.342576 14332 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:02.343060 14332 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:14:02.344317 14013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:02.351044 14363 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:02.355438 14332 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:14:02.362658 14365 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
INSERT report
rows total: 2000
time total: 467.406 ms
time per row: 0.233703 ms
I20240501 08:14:03.031119 12529 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:14:03.031847 12529 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_3 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:03.041040 13182 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.041234 13048 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.041568 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.041669 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.044914 12653 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.045445 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.046092 12784 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.046541 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.046917 12916 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.047302 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.048354 13181 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.048388 12918 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.048731 12918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.048750 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.048725 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.049125 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.049865 12785 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.050244 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.052381 12917 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.052573 13447 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.052783 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.052966 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.053231 13049 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.053637 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.056416 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 389d0cce047e48d7b702cb7245f60659 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.056417 12652 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.056643 12786 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.056852 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.056866 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.056991 12786 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.060350 13313 tablet_service.cc:1467] Processing CreateTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.060422 12651 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.060755 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.061805 13445 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.062244 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.064165 14442 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:03.064392 13183 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.064859 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.065135 13315 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.065492 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.068406 13050 tablet_service.cc:1467] Processing CreateTablet for tablet cd1c9882eb034801aa59324fe069461e (DEFAULT_TABLE table=rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:03.068846 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.076903 14444 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:03.082248 14444 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.085862 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:03.091809 14442 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.094173 14445 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:03.094841 14444 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:03.094944 14450 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:03.095115 14444 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.007s
I20240501 08:14:03.095980 14445 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.096000 14444 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.096469 14444 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.096604 14444 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.097664 14450 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.100469 14444 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.102671 14444 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240501 08:14:03.103438 14444 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:03.104764 14445 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:03.105046 14445 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.000s
I20240501 08:14:03.106169 14445 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.107632 14444 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.109730 14445 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.110222 14445 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.111335 14445 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.111768 14447 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:03.112906 14446 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:03.113090 14450 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:03.114387 14450 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.020s user 0.004s sys 0.000s
I20240501 08:14:03.114670 14447 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.115907 14450 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.116374 14446 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.116976 14443 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:03.118626 14443 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.119117 14444 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:03.119305 14444 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.009s
I20240501 08:14:03.120074 14444 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.118016 14445 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.013s user 0.003s sys 0.000s
I20240501 08:14:03.122066 14444 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.122166 14444 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.122455 14444 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.123713 14445 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:03.125264 14445 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.126873 14444 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:03.126932 14442 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:03.127084 14450 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.127293 14442 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.063s user 0.004s sys 0.004s
I20240501 08:14:03.127413 14450 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.128217 14442 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.128747 14450 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.129587 14450 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.014s user 0.002s sys 0.000s
I20240501 08:14:03.129729 14442 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.130061 14450 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:03.130167 14444 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:03.130194 14442 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.132028 14443 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:03.132140 14442 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.132385 14444 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.133723 14447 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:03.133811 14442 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:14:03.133996 14443 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.002s
I20240501 08:14:03.134191 14442 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:03.134920 14443 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.135413 14447 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.024s user 0.007s sys 0.000s
I20240501 08:14:03.135820 14442 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.136644 14447 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.137048 14443 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.137194 14443 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.137514 14443 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.137760 14450 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.137854 14447 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.138005 14447 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.140157 14443 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:14:03.140412 14447 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.141045 14443 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:03.141523 14447 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:03.143162 13963 raft_consensus.cc:482] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:03.143326 13963 raft_consensus.cc:504] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.143788 14442 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:03.143998 14442 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.001s
I20240501 08:14:03.144519 13963 leader_election.cc:288] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:03.145015 14446 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:03.145237 14446 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.032s user 0.007s sys 0.000s
I20240501 08:14:03.145360 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:03.146031 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
W20240501 08:14:03.146032 13270 leader_election.cc:341] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.146246 14446 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
W20240501 08:14:03.146566 13269 leader_election.cc:341] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.146639 14446 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.146626 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:03.145725 14442 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.146832 14446 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.146924 14442 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.147032 14442 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.147065 14446 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.147274 14442 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.147420 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:03.147629 12936 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:03.147795 14446 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:03.147997 14442 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240501 08:14:03.148108 14446 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
W20240501 08:14:03.148306 13269 leader_election.cc:341] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.148586 14442 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:03.149247 14450 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:03.149412 14450 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.019s user 0.000s sys 0.005s
I20240501 08:14:03.149700 14446 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.150143 14450 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.150425 14450 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.146653 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:03.150496 14450 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.150532 14442 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
W20240501 08:14:03.150975 13271 leader_election.cc:341] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.151082 13271 leader_election.cc:302] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:03.147157 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:03.151363 13963 raft_consensus.cc:2726] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:03.150740 14450 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
W20240501 08:14:03.151780 13270 leader_election.cc:341] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.152608 14447 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:03.154330 14450 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240501 08:14:03.156055 14443 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.156775 14447 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.156930 14450 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:03.158694 14450 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.158957 14442 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:03.159145 14442 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
I20240501 08:14:03.159950 14442 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.160228 14442 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.160323 14442 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.160571 14442 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.161219 14442 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:03.162631 14450 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:03.162801 14450 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.002s
I20240501 08:14:03.163087 14446 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:03.163336 14446 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.000s
I20240501 08:14:03.164213 14444 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:03.164379 14444 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.034s user 0.002s sys 0.009s
I20240501 08:14:03.164669 14450 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.164942 14450 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.165035 14450 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.165117 14444 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.165385 14444 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.164216 14446 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.165308 14450 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.165483 14444 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.165606 14446 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.165593 14351 raft_consensus.cc:482] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:03.165743 14446 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.165753 14351 raft_consensus.cc:504] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.165982 14450 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:14:03.166131 14446 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.166554 14444 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.167011 14351 leader_election.cc:288] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:03.167893 14444 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:03.168102 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:03.168337 13068 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:03.168715 14446 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:14:03.169097 14446 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:03.169368 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:03.169566 13333 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:03.169821 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:03.169847 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:03.170089 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:03.170296 12806 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
W20240501 08:14:03.170351 12873 leader_election.cc:341] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:14:03.170682 12874 leader_election.cc:341] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.170928 12873 leader_election.cc:302] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters: b17fe4a1f54c4655b85eb663fcff1b93, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:03.171260 14351 raft_consensus.cc:2781] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:03.171365 14351 raft_consensus.cc:482] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:03.171455 14351 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.171646 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:03.171891 12671 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:03.172500 14445 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:03.172696 14445 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.049s user 0.003s sys 0.000s
I20240501 08:14:03.172847 14446 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.173506 14445 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.173802 14445 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.173877 14445 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.174145 14445 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.174765 14445 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:03.175115 14445 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:03.175740 14351 raft_consensus.cc:504] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.176733 14445 tablet_bootstrap.cc:654] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.178226 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:03.178460 13069 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.179908 14351 leader_election.cc:288] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Requested vote from peers f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:03.180256 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:03.180509 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:03.180749 13333 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
W20240501 08:14:03.181356 12873 leader_election.cc:341] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.181510 13069 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:03.181890 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:03.182137 12806 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.182163 14446 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:03.182312 14446 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.004s
I20240501 08:14:03.183094 14446 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.183238 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:03.183344 14446 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.183432 14446 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.183440 12671 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.183476 13333 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:03.183701 14446 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.184233 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:03.184448 14446 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:03.184772 12806 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:03.185381 12873 leader_election.cc:302] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters: fa5b68f497b84b31ac35e5549b2af1ad
W20240501 08:14:03.185658 12874 leader_election.cc:341] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:03.185729 14351 raft_consensus.cc:2781] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:03.186941 14351 raft_consensus.cc:686] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Becoming Leader. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Running, Role: LEADER
I20240501 08:14:03.187299 14351 consensus_queue.cc:237] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.187791 12671 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:03.188542 14443 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:03.188541 14447 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:03.188735 14443 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.048s user 0.004s sys 0.000s
I20240501 08:14:03.188799 14447 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.036s user 0.000s sys 0.004s
I20240501 08:14:03.189510 14443 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.189800 14443 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.189653 14447 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.189877 14443 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.190126 14447 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.190258 14447 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.190110 14443 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.190539 14445 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:03.190704 14445 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.000s
I20240501 08:14:03.190696 14443 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:03.191030 14443 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:03.190888 14447 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.191462 14445 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.191725 14445 raft_consensus.cc:374] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.191833 14445 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.192147 14445 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.194588 14447 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:03.194837 14445 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:14:03.195101 14447 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:03.197207 14443 tablet_bootstrap.cc:654] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.199198 14447 tablet_bootstrap.cc:654] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:03.199584 12528 catalog_manager.cc:5484] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:03.201643 14447 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:03.201697 14443 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:03.201834 14443 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.003s
I20240501 08:14:03.201834 14447 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.001s
I20240501 08:14:03.202618 14443 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.202610 14447 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.202898 14447 raft_consensus.cc:374] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.202912 14443 raft_consensus.cc:374] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:03.202998 14443 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.203030 14447 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:03.203262 14447 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.203251 14443 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.203881 14447 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:14:03.203876 14443 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:14:03.223534 14355 raft_consensus.cc:482] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:03.223742 14355 raft_consensus.cc:504] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.225118 14355 leader_election.cc:288] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:03.225208 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:03.225294 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:03.225546 13069 raft_consensus.cc:2379] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fa5b68f497b84b31ac35e5549b2af1ad in current term 1: Already voted for candidate 5bf85222f4734b7885eb866efd27d5f2 in this term.
I20240501 08:14:03.225543 13466 raft_consensus.cc:2454] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:03.225593 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:03.225874 13333 raft_consensus.cc:2379] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fa5b68f497b84b31ac35e5549b2af1ad in current term 1: Already voted for candidate 5bf85222f4734b7885eb866efd27d5f2 in this term.
I20240501 08:14:03.226239 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:03.226300 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:03.226418 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3cbbfb0c743496fb4fddfed91bd6913" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:03.226519 12806 raft_consensus.cc:2379] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fa5b68f497b84b31ac35e5549b2af1ad in current term 1: Already voted for candidate 5bf85222f4734b7885eb866efd27d5f2 in this term.
I20240501 08:14:03.226657 12671 raft_consensus.cc:2379] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fa5b68f497b84b31ac35e5549b2af1ad in current term 1: Already voted for candidate 5bf85222f4734b7885eb866efd27d5f2 in this term.
I20240501 08:14:03.226948 13138 leader_election.cc:302] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, fa5b68f497b84b31ac35e5549b2af1ad; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:03.227252 14355 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.229588 14355 raft_consensus.cc:2726] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:03.262643 13977 raft_consensus.cc:482] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:03.262877 13977 raft_consensus.cc:504] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.264027 13977 leader_election.cc:288] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:03.264549 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:03.264597 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:03.264642 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:03.264674 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:03.264817 13333 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.264817 12806 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.264914 12671 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.264904 13069 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.265517 13402 leader_election.cc:302] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29; no voters:
I20240501 08:14:03.265619 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:03.265856 12936 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.265872 13977 raft_consensus.cc:2781] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:03.265987 13977 raft_consensus.cc:482] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:03.265983 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:03.266074 13977 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.266224 13203 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:03.266716 14291 raft_consensus.cc:482] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:03.266883 14291 raft_consensus.cc:504] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.268046 14291 leader_election.cc:288] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:03.268081 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:03.268321 12806 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.268461 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:03.268462 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:03.268682 13466 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.268699 13333 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.269109 12608 leader_election.cc:302] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29; no voters:
I20240501 08:14:03.269222 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:03.269256 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:03.269421 12936 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.269495 14291 raft_consensus.cc:2781] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:03.269498 13069 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.269594 14291 raft_consensus.cc:482] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:03.269685 14291 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.269711 13977 raft_consensus.cc:504] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.270251 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:03.270510 13203 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:03.271287 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:03.271513 13069 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.271775 13977 leader_election.cc:288] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Requested vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:03.272305 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:03.272506 12936 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.272814 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:03.273047 13203 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.273190 14291 raft_consensus.cc:504] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.274119 13069 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.274379 14291 leader_election.cc:288] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:03.274585 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:03.274786 12937 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.275414 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:03.275408 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:03.275568 13203 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.275617 12805 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.275635 13334 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.276438 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:03.276643 13333 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.277426 12937 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.278299 12805 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.278388 13334 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.278399 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:03.278596 13068 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.278890 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:03.279091 12671 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.279160 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:03.279273 13333 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.279371 13466 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.279618 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "389d0cce047e48d7b702cb7245f60659" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:03.279814 12806 raft_consensus.cc:3037] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.281301 13068 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.281998 13466 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.282097 12671 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.282292 12608 leader_election.cc:302] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:03.282608 13401 leader_election.cc:302] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:03.282925 13977 raft_consensus.cc:2781] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:03.283064 13977 raft_consensus.cc:686] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Becoming Leader. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Running, Role: LEADER
I20240501 08:14:03.283244 12806 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.283432 13977 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:03.283617 14291 raft_consensus.cc:2781] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:03.283763 14291 raft_consensus.cc:686] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Becoming Leader. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Running, Role: LEADER
I20240501 08:14:03.284119 14291 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:03.284658 12936 raft_consensus.cc:2454] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:03.285151 13203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd1c9882eb034801aa59324fe069461e" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:03.285379 13203 raft_consensus.cc:3037] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.288826 13203 raft_consensus.cc:2454] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:03.289427 12525 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: term changed from 0 to 1, leader changed from <none> to b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135). New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:03.289916 12528 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:03.664397 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.688282 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.693332 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.698928 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:03.699220 14476 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
I20240501 08:14:03.699569 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:03.699688 14476 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:14:03.714474 14476 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:14:03.726941 14291 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.731454 14351 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.746802 14401 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.750494 14351 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.757226 14401 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.760039 13977 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.775223 14496 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.777838 14351 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.787915 14401 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240501 08:14:03.789968 13466 raft_consensus.cc:3037] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:03.794452 14351 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.803865 14351 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.809723 14401 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:03.840613 14512 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:03.893800 12873 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 56968.4us
Using table 'rebalance_test_table_3'
INSERT report
rows total: 2000
time total: 416.206 ms
time per row: 0.208103 ms
I20240501 08:14:04.488095 12531 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:14:04.488827 12531 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_4 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:04.498376 12785 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.498517 12651 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.498900 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.498981 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.498982 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.499387 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.501267 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.501681 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.504714 12917 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.505154 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.505275 13048 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.505297 13182 tablet_service.cc:1467] Processing CreateTablet for tablet 8b196f49443f4479837a06e16e7aca6b (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.505672 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.505698 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.513509 13181 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.513998 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.516314 13050 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.516767 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.516979 13315 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.517444 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.533519 14593 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:04.536044 14591 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:04.537498 14589 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:04.537744 14591 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.539090 14592 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:04.540203 14589 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.540768 14592 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.541155 14593 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.543217 13445 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.543620 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.544056 12784 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.544610 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.547286 14590 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:04.550640 14590 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.550901 14591 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:04.551120 14591 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.015s user 0.000s sys 0.003s
I20240501 08:14:04.551745 14592 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:04.551936 14592 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.003s
I20240501 08:14:04.552141 14591 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.552596 14591 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.552753 14591 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.553061 14591 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.553696 14591 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.554744 14591 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:04.555310 14593 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:04.555595 14593 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.022s user 0.004s sys 0.000s
I20240501 08:14:04.555788 13314 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.556274 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.556497 14593 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.556780 14593 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.556893 14593 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.557268 14591 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.557121 14593 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.558941 14593 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:04.559273 14593 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:04.560952 14593 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.563895 13182 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.564365 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.554244 14592 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.566511 14592 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.566642 14592 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.566617 12652 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.567062 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.566905 14592 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.567549 14592 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.014s user 0.001s sys 0.000s
I20240501 08:14:04.568408 13048 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.568848 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.569371 14590 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:04.569551 14590 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.022s user 0.003s sys 0.000s
I20240501 08:14:04.571189 14590 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.571488 14590 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.571568 14590 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.572563 12651 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.572991 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.573051 14600 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:04.573906 14590 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.575289 14596 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:04.575553 14593 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:04.575739 14593 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.017s user 0.000s sys 0.003s
I20240501 08:14:04.576510 14593 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.576823 14593 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.576893 14593 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.576962 14596 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.577143 14593 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.577837 12916 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.578294 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.578500 14600 tablet_bootstrap.cc:654] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.581684 13446 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.582149 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.585260 14592 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:04.587129 14592 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.588414 14590 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.019s user 0.000s sys 0.003s
I20240501 08:14:04.588806 14590 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:04.588939 14591 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:04.589182 14591 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.034s user 0.003s sys 0.002s
I20240501 08:14:04.590015 14591 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.590413 14591 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.590528 14590 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.590536 14591 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.590916 14591 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.592309 14593 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.016s user 0.000s sys 0.003s
I20240501 08:14:04.592379 14589 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:04.592572 14589 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.055s user 0.007s sys 0.000s
I20240501 08:14:04.593432 14592 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:04.593501 14589 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.593685 14592 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.002s
I20240501 08:14:04.593869 14589 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.594033 14589 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.594424 14589 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.594406 14592 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.594703 14592 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.595844 14589 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:04.596158 14600 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:04.596220 14589 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:04.596405 14600 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.023s user 0.004s sys 0.000s
I20240501 08:14:04.597206 14600 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.597550 14592 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.597854 14589 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.597991 14592 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.591574 14591 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:04.599109 14591 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:04.599764 14592 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:04.599906 14600 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.600075 14600 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.600389 14600 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.600970 14592 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:04.601447 14600 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240501 08:14:04.601511 14591 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.603024 14592 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.604882 14590 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:04.605137 14590 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.003s
I20240501 08:14:04.606099 14590 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.606499 14590 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.606611 14590 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.606941 14590 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.607764 14590 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:14:04.612167 14591 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:04.613008 14591 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.014s user 0.004s sys 0.003s
I20240501 08:14:04.614014 14591 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.614415 14591 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.614538 14591 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.614833 14591 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.616180 14591 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:04.620327 14589 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:04.620627 14589 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.024s user 0.006s sys 0.001s
I20240501 08:14:04.621505 14589 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.621888 14592 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:04.621886 14589 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.621964 14589 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.622103 14592 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.021s user 0.000s sys 0.004s
I20240501 08:14:04.622486 14589 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.622910 14592 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.623221 14592 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.623277 14589 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240501 08:14:04.623354 14592 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.623625 14589 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:04.623783 14593 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:04.623690 14592 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.625073 14592 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240501 08:14:04.625447 14589 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.625483 14593 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.627223 14589 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:04.627424 14589 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240501 08:14:04.628252 14589 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.628578 14589 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.628715 14589 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.629025 14589 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.629808 14589 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.633407 14593 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:04.633654 14593 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.004s
I20240501 08:14:04.634567 14593 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.634922 14593 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.635079 14593 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.635387 14593 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.636173 14593 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:04.636376 12784 tablet_service.cc:1467] Processing CreateTablet for tablet b6828130a760491eb21fd3d346219748 (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.637090 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.641559 14590 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:04.641556 12917 tablet_service.cc:1467] Processing CreateTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a (DEFAULT_TABLE table=rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:04.642102 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:04.643095 14590 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.649379 14600 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:04.649938 14596 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:04.650190 14596 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.075s user 0.020s sys 0.032s
I20240501 08:14:04.651203 14600 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.651067 14596 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.651504 14596 raft_consensus.cc:374] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.651643 14596 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.652005 14596 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.652698 14596 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.653029 14596 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:04.653625 14590 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:04.653856 14590 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.012s user 0.008s sys 0.002s
I20240501 08:14:04.654708 14596 tablet_bootstrap.cc:654] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.654708 14590 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.655030 14590 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.655140 14355 raft_consensus.cc:482] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:04.655205 14590 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.655367 14355 raft_consensus.cc:504] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.655537 14590 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.656673 14355 leader_election.cc:288] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:04.658573 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:04.658843 12805 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.659832 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:04.660109 12937 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.660627 14365 raft_consensus.cc:482] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:04.660678 14600 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:04.660849 14600 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.006s
I20240501 08:14:04.660840 14365 raft_consensus.cc:504] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.661335 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:04.661540 13069 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.661612 14600 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.661855 14600 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.661923 14600 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.662127 14365 leader_election.cc:288] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:04.662228 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:04.662158 14600 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.662474 12671 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.662783 14600 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.662904 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:04.663146 14600 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:04.663132 13467 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.663362 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:04.663561 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:04.663769 13334 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:04.663882 14596 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:04.664086 14596 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
W20240501 08:14:04.664305 13006 leader_election.cc:341] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:04.664757 14600 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.665057 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:04.665151 14596 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.665324 12673 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:04.665509 14596 raft_consensus.cc:374] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.665637 14596 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.665928 14596 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.666617 14596 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.666982 14596 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:04.666894 13139 leader_election.cc:302] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:04.667272 14355 raft_consensus.cc:2781] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:04.667374 14355 raft_consensus.cc:482] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:04.667476 14355 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.668072 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:04.668349 13202 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:04.668944 14596 tablet_bootstrap.cc:654] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:04.669765 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:04.670053 12805 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:04.670629 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:04.670895 13334 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:04.671476 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:04.671702 12937 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:04.672456 13005 leader_election.cc:302] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters: b17fe4a1f54c4655b85eb663fcff1b93
I20240501 08:14:04.673867 14355 raft_consensus.cc:504] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.675592 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:04.675729 14355 leader_election.cc:288] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:04.675870 12936 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.676317 14590 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.022s user 0.001s sys 0.001s
I20240501 08:14:04.677790 14596 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:04.678004 14596 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.007s
I20240501 08:14:04.678689 12936 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.679270 14596 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.679598 14596 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.679733 14596 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.680128 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:04.680089 14596 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.680380 12805 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.680900 14596 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:04.681469 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:04.681661 13466 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.683780 13466 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.684582 14365 raft_consensus.cc:2781] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:04.684707 14365 raft_consensus.cc:482] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:04.684818 14365 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.686187 12673 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:04.686419 12673 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.687784 14365 raft_consensus.cc:504] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.689026 12673 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.689045 12805 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.689621 13138 leader_election.cc:302] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:04.689962 14355 raft_consensus.cc:2781] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:04.690135 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:04.690362 13466 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.691259 13333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:04.691623 13333 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.691716 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:04.692176 13069 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:04.692409 13069 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.694120 13466 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.694416 13333 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.694860 13202 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.697032 14600 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:04.697386 14570 raft_consensus.cc:482] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:04.697393 13202 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.697476 14365 leader_election.cc:288] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Requested vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:04.697554 14570 raft_consensus.cc:504] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.698135 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:04.698359 12805 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.698576 14600 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.036s user 0.019s sys 0.007s
I20240501 08:14:04.698820 14570 leader_election.cc:288] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:04.699525 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:04.699755 13466 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:04.699791 14600 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.700861 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:04.700908 13201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:04.701040 12806 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:04.701145 13201 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:04.701690 13270 leader_election.cc:302] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:04.701915 14600 raft_consensus.cc:374] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:04.702039 14600 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:04.702648 14600 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.703212 12670 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:04.703457 12670 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:04.703783 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b196f49443f4479837a06e16e7aca6b" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:04.704010 13334 raft_consensus.cc:3037] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.704006 14600 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:14:04.706614 14355 raft_consensus.cc:686] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Becoming Leader. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Running, Role: LEADER
I20240501 08:14:04.706974 14355 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:04.707384 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:04.707593 12671 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.708555 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8782ed1d2a5492e99b9bf6bd4b1597a" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:04.708742 12938 raft_consensus.cc:3037] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.709571 12937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:04.709728 12937 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:04.711478 12805 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.711783 12938 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.712007 14570 raft_consensus.cc:2781] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:04.712148 14570 raft_consensus.cc:482] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:04.712246 14570 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.712615 13006 leader_election.cc:302] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:04.713032 14620 raft_consensus.cc:2781] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:04.713526 14620 raft_consensus.cc:686] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Becoming Leader. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Running, Role: LEADER
I20240501 08:14:04.714067 14620 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.714560 12671 raft_consensus.cc:2454] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:04.714732 14570 raft_consensus.cc:504] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.717473 13069 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.717592 13201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:04.717784 13201 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.718919 12531 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: term changed from 0 to 1, leader changed from <none> to fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133). New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:04.719614 13334 raft_consensus.cc:2454] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:04.722323 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:04.722550 13070 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.722985 13201 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.722882 12528 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: term changed from 0 to 1, leader changed from <none> to f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132). New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:04.724232 13068 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:04.724848 14570 leader_election.cc:288] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Requested vote from peers f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:04.725131 13070 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.725095 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:04.725385 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:04.725631 12805 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.725679 12671 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.725618 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:04.725819 12936 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.728672 12671 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.729625 13466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6828130a760491eb21fd3d346219748" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:04.729831 13466 raft_consensus.cc:3037] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:04.730469 12936 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.730949 13271 leader_election.cc:302] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:04.731549 14570 raft_consensus.cc:2781] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:04.731752 14570 raft_consensus.cc:686] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Becoming Leader. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Running, Role: LEADER
I20240501 08:14:04.731688 12805 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.732137 14570 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:04.732456 13466 raft_consensus.cc:2454] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:04.735868 12528 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: term changed from 0 to 1, leader changed from <none> to c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134). New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:05.105180 14355 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.116014 14355 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.130229 14570 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.131618 14619 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.134797 14618 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:05.144390 14624 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:05.144912 14624 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:14:05.154430 14570 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:05.154963 14624 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:14:05.159494 14570 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.162454 14619 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.178501 14619 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.190644 14619 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.197636 14620 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.198988 14620 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.206127 14620 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.234419 14620 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.248478 14570 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.284391 14620 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.313019 14570 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:05.332667 14667 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
INSERT report
rows total: 2000
time total: 397.909 ms
time per row: 0.198954 ms
I20240501 08:14:05.899235 12529 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:14:05.900058 12529 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_5 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:05.909533 13314 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.910005 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.910387 12651 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.910640 13048 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.910810 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.911089 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.912616 12784 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.913054 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.913915 12785 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.914322 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.915511 13181 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.915998 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.916316 13182 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.916659 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.917734 13049 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.918179 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.919210 12653 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.919658 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.920341 13050 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.920732 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.921981 13315 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.922070 13183 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.922411 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.922492 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.924361 13445 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.924381 13313 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.924793 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.924999 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.926267 12916 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.926357 13446 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.926695 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.926709 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.928342 13447 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.928752 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.932435 12917 tablet_service.cc:1467] Processing CreateTablet for tablet baba1d19f76a434b858e635590a50c7c (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.932854 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.936401 12652 tablet_service.cc:1467] Processing CreateTablet for tablet e9b18b849c914fa5ba731892b7b88d6e (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.936856 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.939508 14735 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:05.944572 14735 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.946437 14733 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:05.951368 14733 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.952952 14732 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:05.959337 14732 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.965176 12784 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.966003 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.968916 12917 tablet_service.cc:1467] Processing CreateTablet for tablet cab182a0e126421c986101ea19df3e97 (DEFAULT_TABLE table=rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:05.969429 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:05.980538 14738 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:05.982385 14738 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.982870 14732 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:05.983831 14732 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.031s user 0.006s sys 0.000s
I20240501 08:14:05.984292 14739 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:05.985802 14732 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:05.986645 14740 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:05.987327 14734 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:05.987561 14733 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:05.987769 14733 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.041s user 0.002s sys 0.008s
I20240501 08:14:05.988613 14733 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:05.988981 14732 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:05.989138 14732 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:05.989629 14734 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.989488 14732 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:05.989990 14739 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.991562 14740 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.997146 14739 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:05.997424 14739 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240501 08:14:05.997635 14738 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:05.997864 14738 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.000s
I20240501 08:14:05.998692 14733 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:05.999053 14732 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.015s user 0.002s sys 0.000s
I20240501 08:14:05.999224 14733 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:05.999583 14738 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:05.999931 14738 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.000130 14738 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:05.998406 14739 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.000406 14739 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.000541 14739 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.000407 14738 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.000871 14739 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.002377 14739 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:14:06.004155 14739 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:06.006259 14739 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:05.999608 14733 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.005147 14732 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:06.008244 14732 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.008889 14733 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.021s user 0.002s sys 0.000s
I20240501 08:14:06.009924 14738 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.012s user 0.001s sys 0.003s
I20240501 08:14:06.010416 14735 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:06.010740 14735 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.071s user 0.009s sys 0.003s
I20240501 08:14:06.011564 14735 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.011938 14735 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.012956 14734 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:06.013159 14734 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.026s user 0.006s sys 0.000s
I20240501 08:14:06.013489 14735 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.013849 14735 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.014230 14734 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.014559 14734 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.014693 14734 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.014972 14734 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.015597 14735 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240501 08:14:06.015635 14734 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:06.015940 14735 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:06.015946 14734 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:06.016359 14738 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:06.018146 14738 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.024878 14733 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:06.028425 14735 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.028437 14734 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.026980 14663 raft_consensus.cc:482] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:06.028702 14663 raft_consensus.cc:504] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.029987 14663 leader_election.cc:288] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.031224 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:06.032727 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:06.033216 14734 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:06.033412 14734 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.004s
W20240501 08:14:06.033412 13006 leader_election.cc:341] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:06.033550 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:06.033818 13467 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.034173 14734 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.034446 14734 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.034534 14734 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.034494 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:06.034765 14734 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.034879 13201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
W20240501 08:14:06.034999 13005 leader_election.cc:341] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:14:06.035539 13004 leader_election.cc:341] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): Illegal state: must be running to vote when last-logged opid is not known
W20240501 08:14:06.035816 13005 leader_election.cc:341] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:06.035909 13005 leader_election.cc:302] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d; no voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:06.036361 14663 raft_consensus.cc:2726] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240501 08:14:06.036849 14738 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:06.037025 14738 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.002s
I20240501 08:14:06.037777 14740 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:06.037740 14738 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.037953 14740 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.053s user 0.002s sys 0.007s
I20240501 08:14:06.038007 14738 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.038108 14738 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.038363 14738 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.038928 14738 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:14:06.039548 14734 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:06.039893 14734 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:06.040915 14733 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.035395 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:06.041435 14734 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.043385 14740 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.043699 14740 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.043781 14740 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.044329 14740 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.045033 14733 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:06.045202 14733 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.035s user 0.001s sys 0.002s
W20240501 08:14:06.045388 13005 leader_election.cc:341] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:06.045872 14740 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.008s user 0.000s sys 0.003s
I20240501 08:14:06.046007 14733 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.046288 14733 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.046357 14733 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.046619 14733 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.046238 14740 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:06.047394 14733 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:06.047698 14733 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:06.050503 14740 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.050598 14733 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.051512 14735 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:06.051792 14735 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.036s user 0.002s sys 0.001s
I20240501 08:14:06.052150 14739 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:06.052335 14739 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.048s user 0.019s sys 0.018s
I20240501 08:14:06.052829 14735 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.053207 14735 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.053062 14739 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.053359 14739 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.053373 14735 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.053443 14739 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.053755 14735 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.053738 14739 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.054633 14735 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:06.054960 14735 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:06.055145 14739 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:06.055428 14739 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:06.057010 14739 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.057885 14735 tablet_bootstrap.cc:654] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.064546 14740 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:06.064769 14740 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.019s user 0.000s sys 0.004s
I20240501 08:14:06.065989 14740 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.066304 14740 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.066396 14740 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.067764 14734 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:06.067999 14734 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.028s user 0.003s sys 0.003s
I20240501 08:14:06.068868 14734 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.069449 14734 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.069573 14734 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.069882 14734 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.070673 14734 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:06.072865 14740 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.073642 14740 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.009s user 0.000s sys 0.003s
I20240501 08:14:06.074029 14740 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:06.075726 14740 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.092873 14738 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:06.095146 14738 tablet_bootstrap.cc:654] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.098536 14732 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:06.098767 14732 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.094s user 0.004s sys 0.000s
I20240501 08:14:06.099625 14732 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.099943 14732 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.100337 14732 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.100607 14732 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.101295 14732 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:06.103193 14732 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:06.105031 14732 tablet_bootstrap.cc:654] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:06.105974 14738 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:06.106262 14738 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.001s
I20240501 08:14:06.107218 14738 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.108124 14738 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.108289 14738 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.108671 14738 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.113408 14739 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:06.113408 14732 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:06.113586 14739 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.058s user 0.016s sys 0.014s
I20240501 08:14:06.113586 14732 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.005s
I20240501 08:14:06.114346 14732 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.114346 14739 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.114629 14732 raft_consensus.cc:374] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.114626 14739 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.114712 14732 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.114713 14739 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.114979 14739 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.116550 14739 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:14:06.114949 14732 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.117677 14732 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.004s user 0.002s sys 0.001s
I20240501 08:14:06.120471 14738 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.014s user 0.002s sys 0.001s
I20240501 08:14:06.129232 14735 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:06.129422 14735 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.075s user 0.008s sys 0.025s
I20240501 08:14:06.129873 14740 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:06.130045 14740 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.056s user 0.021s sys 0.032s
I20240501 08:14:06.130095 14735 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.130352 14735 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.130443 14735 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.130772 14740 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.130828 14735 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.131028 14740 raft_consensus.cc:374] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.131107 14740 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.131464 14735 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:06.131363 14740 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.135264 14663 raft_consensus.cc:482] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:06.135460 14663 raft_consensus.cc:504] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.136791 14663 leader_election.cc:288] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.138702 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:06.138943 12671 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.132082 14740 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:06.139537 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:06.139793 13467 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.140262 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:06.140486 13334 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.140902 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:06.141152 12806 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.142784 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:06.143015 12936 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.143533 14733 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:06.143724 14733 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.096s user 0.019s sys 0.015s
I20240501 08:14:06.144657 14733 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.144953 14733 raft_consensus.cc:374] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:06.145051 14733 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:06.145306 14733 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.146093 14733 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240501 08:14:06.148459 13201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:06.148696 13201 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 0.
I20240501 08:14:06.149330 13006 leader_election.cc:302] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:06.149617 14663 raft_consensus.cc:2781] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:06.149730 14663 raft_consensus.cc:482] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:06.149806 14663 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.152874 14663 raft_consensus.cc:504] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.154229 13201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:06.154448 13201 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.154459 12805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:06.154515 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:06.154672 12805 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.154727 13334 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.154984 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:06.155190 12671 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.157320 12805 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.157550 13334 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.157866 12671 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.158370 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:06.158620 13467 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.158840 14663 leader_election.cc:288] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Requested vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.159334 13005 leader_election.cc:302] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:06.160117 14663 raft_consensus.cc:2781] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:06.160286 14663 raft_consensus.cc:686] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Becoming Leader. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Running, Role: LEADER
I20240501 08:14:06.160802 13467 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.160612 14663 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.161713 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b18b849c914fa5ba731892b7b88d6e" candidate_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:06.161940 12936 raft_consensus.cc:3037] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.162601 13201 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.164304 12936 raft_consensus.cc:2454] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f59d8b4fd79049b6b40dcecd13d0be9d in term 1.
I20240501 08:14:06.172212 12528 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: term changed from 0 to 1, leader changed from <none> to f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132). New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:06.209409 14619 raft_consensus.cc:482] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:06.209694 14619 raft_consensus.cc:504] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.211071 14619 leader_election.cc:288] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.211310 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:06.211308 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:06.211573 13070 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.211575 13334 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.212038 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:06.212260 13467 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.212334 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:06.212586 12671 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.212471 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:06.212981 12936 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.213078 13137 leader_election.cc:302] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:06.213534 14619 raft_consensus.cc:2781] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:06.213644 14619 raft_consensus.cc:482] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:06.213735 14619 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.213829 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:06.214073 12806 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 0.
I20240501 08:14:06.216002 14619 raft_consensus.cc:504] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.217034 14619 leader_election.cc:288] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Requested vote from peers c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.217432 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:06.217634 13467 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.217877 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:06.218097 12671 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.218034 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:06.218248 13334 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.220912 13334 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.221295 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:06.221563 13070 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.222358 13467 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.222736 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:06.222970 12936 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.224308 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "baba1d19f76a434b858e635590a50c7c" candidate_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:06.224539 12806 raft_consensus.cc:3037] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.224617 13070 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.225136 13137 leader_election.cc:302] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:06.225669 14619 raft_consensus.cc:2781] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:06.225950 14619 raft_consensus.cc:686] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Becoming Leader. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Running, Role: LEADER
I20240501 08:14:06.226425 12936 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.226450 14619 consensus_queue.cc:237] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.228837 12671 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.229808 12806 raft_consensus.cc:2454] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa5b68f497b84b31ac35e5549b2af1ad in term 1.
I20240501 08:14:06.231954 12528 catalog_manager.cc:5484] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: term changed from 0 to 1, leader changed from <none> to fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133). New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:06.293612 14668 raft_consensus.cc:482] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:06.293820 14668 raft_consensus.cc:504] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.294939 14668 leader_election.cc:288] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.295207 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:06.295467 13467 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.295775 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:06.295912 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:06.296054 12806 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.296177 13202 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.296058 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:06.296376 13070 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.295272 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:06.296819 13271 leader_election.cc:302] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:06.296979 12671 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.297258 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:06.297396 14668 raft_consensus.cc:2781] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:06.297518 14668 raft_consensus.cc:482] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:06.297493 12936 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 0.
I20240501 08:14:06.297617 14668 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.300631 14668 raft_consensus.cc:504] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.302474 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:06.302574 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:06.302734 13070 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.302824 13202 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.305037 13070 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.305140 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:06.305361 12806 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.305660 13202 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.306677 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:06.306875 12671 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.306869 13467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:06.307102 13467 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.307547 12806 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.308558 13270 leader_election.cc:302] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:06.309644 12936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab182a0e126421c986101ea19df3e97" candidate_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:06.309856 12936 raft_consensus.cc:3037] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:06.309883 13467 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.310361 12671 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.310704 14763 raft_consensus.cc:2781] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:06.312753 12936 raft_consensus.cc:2454] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c58c4517fd6941f19a9e75d1e48e8b29 in term 1.
I20240501 08:14:06.313365 14668 leader_election.cc:288] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [CANDIDATE]: Term 1 election: Requested vote from peers fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:06.314078 14763 raft_consensus.cc:686] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Becoming Leader. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Running, Role: LEADER
I20240501 08:14:06.314411 14763 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:06.318388 12528 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: term changed from 0 to 1, leader changed from <none> to c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134). New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:06.586328 14663 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.636946 14663 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.639101 14726 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.646800 14726 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.647332 14726 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.661259 14663 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.678686 14636 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.684504 14636 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.702514 14763 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.714016 14636 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.719950 14636 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.723783 14636 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.728260 14619 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.756451 14763 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.793181 14763 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.802912 14763 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.807607 14794 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:06.810304 14763 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:06.848731 14766 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:06.849081 14766 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:14:06.856776 14766 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
INSERT report
rows total: 2000
time total: 459.612 ms
time per row: 0.229806 ms
I20240501 08:14:07.493264 12522 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33332:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240501 08:14:07.494022 12522 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_6 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240501 08:14:07.503778 12653 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.503774 12784 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.504276 12653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.504285 12784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.509260 13048 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.509310 13050 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.509654 12651 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.509770 13048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.509980 13050 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.510030 12651 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.510079 13182 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.510468 13182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.512328 13049 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.512322 12652 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.512663 12652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.512687 13049 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.516407 13181 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.516434 13313 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.516805 13181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.516870 13313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.520380 13314 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.520398 13183 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.520777 13314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.520810 13183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.524380 13315 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.524793 13315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.525713 12785 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.526142 12785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.528416 12786 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.528846 12786 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.533464 13446 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.533804 13447 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.533955 13446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.533892 12916 tablet_service.cc:1467] Processing CreateTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.534075 12918 tablet_service.cc:1467] Processing CreateTablet for tablet 1226795e4f534366874b23270d086334 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.534200 13447 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.534284 12916 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.533488 13445 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.534498 12918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.534770 13445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.540401 12917 tablet_service.cc:1467] Processing CreateTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 (DEFAULT_TABLE table=rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240501 08:14:07.540854 12917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:07.562788 14883 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:07.563201 14886 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:07.565701 14883 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.569643 14887 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:07.570290 14888 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:07.570883 14884 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:07.573151 14886 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.573693 14888 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.574172 14884 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.575556 14887 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.575551 14882 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:07.577724 14882 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.583186 14886 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:07.584664 14885 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:07.584944 14886 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.022s user 0.003s sys 0.004s
I20240501 08:14:07.585135 14888 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:07.585361 14888 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.000s
I20240501 08:14:07.585896 14882 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:07.585944 14886 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.586176 14888 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.586707 14884 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:07.586895 14884 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.003s
I20240501 08:14:07.587108 14887 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:07.587313 14887 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.018s user 0.005s sys 0.000s
I20240501 08:14:07.587620 14884 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.588066 14887 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.588369 14887 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.588474 14884 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.588495 14887 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.588660 14884 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.588769 14887 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.589108 14888 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.589004 14884 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.589619 14887 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:07.589763 14888 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.589780 14886 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.589895 14886 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.590027 14888 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.590176 14886 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.590942 14885 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.591137 14884 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240501 08:14:07.591503 14884 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:07.591503 14883 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:07.591662 14883 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.029s user 0.002s sys 0.004s
I20240501 08:14:07.592059 14887 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:07.592132 14882 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.000s
I20240501 08:14:07.592486 14883 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.592808 14883 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.592939 14883 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.593227 14883 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.593911 14883 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:07.594215 14882 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.595484 14884 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.595608 14882 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.595759 14882 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.596081 14882 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.593237 14888 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240501 08:14:07.599936 14882 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240501 08:14:07.600358 14882 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:07.600674 14883 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:07.601382 14888 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:07.602272 14883 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.602852 14888 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.603684 14887 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.606727 14882 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.610673 14886 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.026s user 0.002s sys 0.002s
I20240501 08:14:07.610884 14885 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:07.611171 14885 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.027s user 0.002s sys 0.004s
I20240501 08:14:07.611471 14883 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:07.611726 14883 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.005s
I20240501 08:14:07.612295 14882 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:07.612773 14882 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.004s
I20240501 08:14:07.613740 14885 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.613906 14882 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.614146 14885 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.614257 14885 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.614689 14885 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.615410 14882 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.615824 14882 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.615943 14888 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:07.616216 14888 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.015s user 0.007s sys 0.001s
I20240501 08:14:07.616226 14882 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.617193 14884 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:07.617249 14886 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:07.617481 14884 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.026s user 0.006s sys 0.000s
I20240501 08:14:07.617619 14888 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.617977 14888 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.618094 14888 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.618271 14885 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240501 08:14:07.618309 14884 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.618484 14888 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.618710 14884 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.618782 14884 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.618914 14886 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.619050 14884 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.619786 14887 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:07.620054 14887 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.028s user 0.005s sys 0.002s
I20240501 08:14:07.620431 14884 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:07.620740 14884 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Bootstrap starting.
I20240501 08:14:07.620649 14883 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.621008 14887 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.621345 14887 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.621477 14887 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.621968 14883 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.621811 14887 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.622121 14883 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.622416 14883 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.624392 14884 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.624699 14887 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:14:07.625018 14887 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:07.625972 14885 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:07.626171 14883 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:07.627550 14888 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.011s user 0.000s sys 0.002s
I20240501 08:14:07.627914 14882 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.015s user 0.002s sys 0.000s
I20240501 08:14:07.628126 14888 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6: Bootstrap starting.
I20240501 08:14:07.628357 14882 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:07.629465 14887 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.630388 14883 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29: Bootstrap starting.
I20240501 08:14:07.633139 14882 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.633162 14888 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.633848 14885 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.634502 14883 tablet_bootstrap.cc:654] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.642547 14886 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:07.642905 14886 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.026s user 0.001s sys 0.007s
I20240501 08:14:07.643424 14885 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:07.643625 14885 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.003s
I20240501 08:14:07.643992 14886 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.644455 14886 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.644572 14886 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.644367 14885 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.645166 14885 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.645246 14885 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.645558 14885 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.646052 14888 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6: No bootstrap required, opened a new log
I20240501 08:14:07.646209 14888 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.007s
I20240501 08:14:07.646314 14885 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:14:07.646971 14888 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.647593 14888 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.647738 14888 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.647922 14885 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Bootstrap starting.
I20240501 08:14:07.648144 14888 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.649333 14882 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: No bootstrap required, opened a new log
I20240501 08:14:07.649535 14882 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.003s
I20240501 08:14:07.649718 14888 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240501 08:14:07.650907 14887 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad: No bootstrap required, opened a new log
I20240501 08:14:07.650306 14882 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.652125 14887 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.027s user 0.007s sys 0.001s
I20240501 08:14:07.652849 14885 tablet_bootstrap.cc:654] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.653292 14887 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.653647 14887 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.653772 14887 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.654043 14887 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.654587 14884 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: No bootstrap required, opened a new log
I20240501 08:14:07.654747 14884 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Time spent bootstrapping tablet: real 0.034s user 0.006s sys 0.000s
I20240501 08:14:07.655377 14488 raft_consensus.cc:482] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:07.644938 14886 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.655514 14884 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.655552 14488 raft_consensus.cc:504] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.655781 14884 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.655874 14884 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.656121 14884 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.656982 14488 leader_election.cc:288] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:07.657465 14884 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:07.652233 14882 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.657671 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:07.657804 14882 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.657886 12938 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:07.658068 14882 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.658298 14886 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.015s user 0.000s sys 0.002s
I20240501 08:14:07.658246 13465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:07.658461 13465 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:07.658871 14886 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:07.658820 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:07.659823 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:07.660058 12806 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:07.660279 14882 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.011s user 0.002s sys 0.000s
W20240501 08:14:07.660910 12608 leader_election.cc:341] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699): Illegal state: must be running to vote when last-logged opid is not known
I20240501 08:14:07.662276 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:07.662525 13070 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:07.663287 12607 leader_election.cc:302] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, b17fe4a1f54c4655b85eb663fcff1b93, f59d8b4fd79049b6b40dcecd13d0be9d; no voters: c58c4517fd6941f19a9e75d1e48e8b29
I20240501 08:14:07.663848 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:07.664227 13202 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 0.
I20240501 08:14:07.665076 14488 raft_consensus.cc:2781] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:07.665199 14488 raft_consensus.cc:482] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:07.665287 14488 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.666417 14886 tablet_bootstrap.cc:654] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9: Neither blocks nor log segments found. Creating new log.
I20240501 08:14:07.668530 14883 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29: No bootstrap required, opened a new log
I20240501 08:14:07.668782 14883 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29: Time spent bootstrapping tablet: real 0.038s user 0.004s sys 0.001s
I20240501 08:14:07.669106 14488 raft_consensus.cc:504] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.669771 14883 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.670125 14883 raft_consensus.cc:374] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.670257 14883 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c58c4517fd6941f19a9e75d1e48e8b29, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.670612 14883 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.671998 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:07.672211 12806 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.674862 12806 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.675529 14883 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240501 08:14:07.678632 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:07.678889 12938 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.678891 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:07.679090 13334 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.679196 14886 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9: No bootstrap required, opened a new log
I20240501 08:14:07.679404 14886 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.021s user 0.000s sys 0.004s
I20240501 08:14:07.680261 14886 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.680606 14886 raft_consensus.cc:374] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.680745 14886 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.681064 14886 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.681324 14885 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: No bootstrap required, opened a new log
I20240501 08:14:07.681514 14885 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent bootstrapping tablet: real 0.035s user 0.004s sys 0.015s
I20240501 08:14:07.681777 13334 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.682060 14886 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:07.682348 14885 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.682606 14885 raft_consensus.cc:374] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240501 08:14:07.682711 14885 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Initialized, Role: FOLLOWER
I20240501 08:14:07.682845 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:07.682972 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:07.683072 13070 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.682952 14885 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.683172 13202 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.684368 14885 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240501 08:14:07.685722 13070 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.686076 13202 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.686363 12607 leader_election.cc:302] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:07.687273 12938 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.687880 14910 raft_consensus.cc:2781] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:07.688463 14488 leader_election.cc:288] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:07.689110 13465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ff1e2f7acd4e0784fef633bd0fd64d" candidate_uuid: "56e1ccecd51040458c27911fa34a90a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:07.689291 13465 raft_consensus.cc:3037] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.691352 13465 raft_consensus.cc:2454] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56e1ccecd51040458c27911fa34a90a6 in term 1.
I20240501 08:14:07.691730 14910 raft_consensus.cc:686] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Becoming Leader. State: Replica: 56e1ccecd51040458c27911fa34a90a6, State: Running, Role: LEADER
I20240501 08:14:07.692273 14887 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.040s user 0.002s sys 0.000s
I20240501 08:14:07.692130 14910 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.695075 14721 raft_consensus.cc:482] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:07.695250 14721 raft_consensus.cc:504] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.696149 12528 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129). New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:07.696533 14721 leader_election.cc:288] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:07.696816 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:07.697012 12671 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.697304 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:07.697379 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:07.697536 13202 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.697602 13070 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.697566 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:07.697626 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:07.697775 13334 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.697840 12806 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.698295 12873 leader_election.cc:302] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29, fa5b68f497b84b31ac35e5549b2af1ad; no voters:
I20240501 08:14:07.698333 13465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" is_pre_election: true
I20240501 08:14:07.698534 14721 raft_consensus.cc:2781] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:07.698551 13465 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 0.
I20240501 08:14:07.698635 14721 raft_consensus.cc:482] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:07.698712 14721 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.701243 14721 raft_consensus.cc:504] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.702370 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:07.702626 12806 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.702670 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:07.702852 12671 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.703600 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:07.703811 13202 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.703814 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:07.704015 13334 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.705415 12806 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.705426 12671 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.706501 13202 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.706516 13334 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.706846 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:07.707029 13070 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.707131 14721 leader_election.cc:288] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699), b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:07.707481 13465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1226795e4f534366874b23270d086334" candidate_uuid: "5bf85222f4734b7885eb866efd27d5f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b17fe4a1f54c4655b85eb663fcff1b93"
I20240501 08:14:07.707651 13465 raft_consensus.cc:3037] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.708428 12873 leader_election.cc:302] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, 56e1ccecd51040458c27911fa34a90a6, 5bf85222f4734b7885eb866efd27d5f2, c58c4517fd6941f19a9e75d1e48e8b29; no voters:
I20240501 08:14:07.708750 14721 raft_consensus.cc:2781] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:07.708927 14721 raft_consensus.cc:686] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Becoming Leader. State: Replica: 5bf85222f4734b7885eb866efd27d5f2, State: Running, Role: LEADER
I20240501 08:14:07.709285 13070 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.709283 14721 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.709950 13465 raft_consensus.cc:2454] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5bf85222f4734b7885eb866efd27d5f2 in term 1.
I20240501 08:14:07.713145 12528 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131). New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:07.725024 14877 raft_consensus.cc:482] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240501 08:14:07.725221 14877 raft_consensus.cc:504] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.726449 14877 leader_election.cc:288] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:07.726518 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" is_pre_election: true
I20240501 08:14:07.726765 12806 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.726933 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" is_pre_election: true
I20240501 08:14:07.727133 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" is_pre_election: true
I20240501 08:14:07.727186 13070 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.727372 13334 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.727756 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6" is_pre_election: true
I20240501 08:14:07.727954 12671 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.728003 13401 leader_election.cc:302] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:07.728003 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2" is_pre_election: true
I20240501 08:14:07.728256 14877 raft_consensus.cc:2781] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240501 08:14:07.728367 14877 raft_consensus.cc:482] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240501 08:14:07.728452 14877 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.728440 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" is_pre_election: true
I20240501 08:14:07.728655 13202 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.728935 12938 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 0.
I20240501 08:14:07.731073 14877 raft_consensus.cc:504] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.732326 12806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9"
I20240501 08:14:07.732591 12806 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.732620 14877 leader_election.cc:288] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Requested vote from peers 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135), 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349), 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689), fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529), f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293), c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:07.732998 13070 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d"
I20240501 08:14:07.733016 13202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa5b68f497b84b31ac35e5549b2af1ad"
I20240501 08:14:07.733206 13202 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.733230 13070 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.733180 13334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c58c4517fd6941f19a9e75d1e48e8b29"
I20240501 08:14:07.733419 13334 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.732340 12671 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56e1ccecd51040458c27911fa34a90a6"
I20240501 08:14:07.734030 12671 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.735847 13070 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.735870 12806 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.736171 13334 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.732396 12938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "16288057e4f24e8ebe4fd959d026d9d8" candidate_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5bf85222f4734b7885eb866efd27d5f2"
I20240501 08:14:07.737306 12938 raft_consensus.cc:3037] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 0 FOLLOWER]: Advancing to term 1
I20240501 08:14:07.737313 13401 leader_election.cc:302] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 1be03ffeb3384b42bfee6ffaf3f451c9, b17fe4a1f54c4655b85eb663fcff1b93, c58c4517fd6941f19a9e75d1e48e8b29, f59d8b4fd79049b6b40dcecd13d0be9d; no voters:
I20240501 08:14:07.737633 14877 raft_consensus.cc:2781] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Leader election won for term 1
I20240501 08:14:07.737777 14877 raft_consensus.cc:686] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Becoming Leader. State: Replica: b17fe4a1f54c4655b85eb663fcff1b93, State: Running, Role: LEADER
I20240501 08:14:07.737874 12671 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.738157 14877 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:07.739444 13202 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.740552 12938 raft_consensus.cc:2454] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b17fe4a1f54c4655b85eb663fcff1b93 in term 1.
I20240501 08:14:07.742449 12528 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: term changed from 0 to 1, leader changed from <none> to b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135). New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:08.093042 14721 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.112406 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.117935 14877 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:08.124753 14916 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:08.125198 14916 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240501 08:14:08.126878 14721 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240501 08:14:08.136236 14916 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240501 08:14:08.153795 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.159477 14877 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_6'
I20240501 08:14:08.185653 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.193210 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.193758 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.195235 14912 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.218210 14721 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.251345 14877 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.260782 14959 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.270830 14877 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.294603 14959 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.305454 14877 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.317699 14959 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240501 08:14:08.341449 14512 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 532.391 ms
time per row: 0.266196 ms
-----------------------------------------------------
Suppressions used:
count bytes template
4 1192 libcrypto.so.1.1
-----------------------------------------------------
I20240501 08:14:09.733733 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.137:37989
--local_ip_for_outbound_sockets=127.28.1.137
--tserver_master_addrs=127.28.1.190:37815
--webserver_port=37645
--webserver_interface=127.28.1.137
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:14:10.107699 15019 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:10.108594 15019 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:14:10.108753 15019 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:14:10.116845 15019 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:14:10.117008 15019 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:14:10.117180 15019 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.137
I20240501 08:14:10.127645 15019 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.137:37989
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.28.1.137
--webserver_port=37645
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.137
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:14:10.129040 15019 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:14:10.131377 15019 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:14:10.148016 15026 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:14:10.150660 15027 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:14:10.153431 15019 server_base.cc:1034] running on GCE node
W20240501 08:14:10.154414 15029 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:14:10.155500 15019 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:14:10.158525 15019 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:14:10.159803 15019 hybrid_clock.cc:648] HybridClock initialized: now 1714551250159735 us; error 60 us; skew 500 ppm
I20240501 08:14:10.160360 15019 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:14:10.163758 15019 webserver.cc:460] Webserver started at http://127.28.1.137:37645/ using document root <none> and password file <none>
I20240501 08:14:10.164821 15019 fs_manager.cc:359] Metadata directory not provided
I20240501 08:14:10.164985 15019 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:14:10.171574 15019 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.002s sys 0.003s
I20240501 08:14:10.174615 15035 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.176195 15019 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:14:10.176367 15019 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "8e309f48ce254660915fb9fc32649ab2"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:14:10.177214 15019 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:14:10.193957 15019 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:14:10.194924 15019 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:14:10.195245 15019 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:14:10.196441 15019 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:14:10.198045 15019 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:14:10.198125 15019 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.198249 15019 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:14:10.198298 15019 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.229768 15019 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.137:37989
I20240501 08:14:10.229801 15147 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.137:37989 every 8 connection(s)
I20240501 08:14:10.231333 15019 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240501 08:14:10.234086 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 15019
I20240501 08:14:10.234899 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.138:40307
--local_ip_for_outbound_sockets=127.28.1.138
--tserver_master_addrs=127.28.1.190:37815
--webserver_port=38787
--webserver_interface=127.28.1.138
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240501 08:14:10.247437 15148 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:14:10.247792 15148 heartbeater.cc:464] Registering TS with master...
I20240501 08:14:10.248746 15148 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:14:10.250516 12522 ts_manager.cc:194] Re-registered known tserver with Master: 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989)
I20240501 08:14:10.251490 12522 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.137:53771
W20240501 08:14:10.609714 15152 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:10.610008 15152 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:14:10.610153 15152 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:14:10.618701 15152 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:14:10.618861 15152 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:14:10.619019 15152 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.138
I20240501 08:14:10.628515 15152 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.138:40307
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.28.1.138
--webserver_port=38787
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.138
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:14:10.629935 15152 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:14:10.632330 15152 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:14:10.645985 15160 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:14:10.646510 15162 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:14:10.646813 15159 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:14:10.647279 15152 server_base.cc:1034] running on GCE node
I20240501 08:14:10.647899 15152 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:14:10.648936 15152 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:14:10.650143 15152 hybrid_clock.cc:648] HybridClock initialized: now 1714551250650084 us; error 45 us; skew 500 ppm
I20240501 08:14:10.650467 15152 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:14:10.657429 15152 webserver.cc:460] Webserver started at http://127.28.1.138:38787/ using document root <none> and password file <none>
I20240501 08:14:10.658296 15152 fs_manager.cc:359] Metadata directory not provided
I20240501 08:14:10.658419 15152 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:14:10.663868 15152 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.002s sys 0.001s
I20240501 08:14:10.666786 15168 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.668550 15152 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240501 08:14:10.668710 15152 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "3cd5ec97de8642dd8796fae63ea73e5a"
format_stamp: "Formatted at 2024-05-01 08:13:54 on dist-test-slave-jj4n"
I20240501 08:14:10.669596 15152 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:14:10.694628 15152 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:14:10.695487 15152 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:14:10.695749 15152 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:14:10.697157 15152 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:14:10.698989 15152 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:14:10.699070 15152 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.699209 15152 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:14:10.699271 15152 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:10.733418 15152 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.138:40307
I20240501 08:14:10.733479 15281 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.138:40307 every 8 connection(s)
I20240501 08:14:10.735073 15152 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240501 08:14:10.738175 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 15152
I20240501 08:14:10.749022 15282 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:14:10.749397 15282 heartbeater.cc:464] Registering TS with master...
I20240501 08:14:10.750420 15282 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:14:10.752040 12522 ts_manager.cc:194] Re-registered known tserver with Master: 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307)
I20240501 08:14:10.752921 12522 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.138:47477
I20240501 08:14:10.780393 28678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
/tmp/dist-test-taskVWMLaH/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.28.1.136:37957
--local_ip_for_outbound_sockets=127.28.1.136
--tserver_master_addrs=127.28.1.190:37815
--webserver_port=42427
--webserver_interface=127.28.1.136
--builtin_ntp_servers=127.28.1.148:40105
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240501 08:14:11.150418 15285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240501 08:14:11.150727 15285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240501 08:14:11.150871 15285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240501 08:14:11.158674 15285 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240501 08:14:11.158808 15285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240501 08:14:11.158954 15285 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.1.136
I20240501 08:14:11.168676 15285 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.28.1.148:40105
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.28.1.136:37957
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.28.1.136
--webserver_port=42427
--tserver_master_addrs=127.28.1.190:37815
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.28.1.136
--log_dir=/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
build id 71
ASAN enabled
I20240501 08:14:11.170274 15285 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240501 08:14:11.172307 15285 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240501 08:14:11.186733 15295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:14:11.187027 15292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240501 08:14:11.187559 15293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240501 08:14:11.187995 15285 server_base.cc:1034] running on GCE node
I20240501 08:14:11.188628 15285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240501 08:14:11.189602 15285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240501 08:14:11.190806 15285 hybrid_clock.cc:648] HybridClock initialized: now 1714551251190738 us; error 61 us; skew 500 ppm
I20240501 08:14:11.191133 15285 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240501 08:14:11.193727 15285 webserver.cc:460] Webserver started at http://127.28.1.136:42427/ using document root <none> and password file <none>
I20240501 08:14:11.194684 15285 fs_manager.cc:359] Metadata directory not provided
I20240501 08:14:11.194833 15285 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240501 08:14:11.200814 15285 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.002s sys 0.001s
I20240501 08:14:11.204088 15301 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:11.205556 15285 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
I20240501 08:14:11.205703 15285 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "35cb9fadf231408598c3877837a6149e"
format_stamp: "Formatted at 2024-05-01 08:13:53 on dist-test-slave-jj4n"
I20240501 08:14:11.206712 15285 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240501 08:14:11.249972 15285 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240501 08:14:11.250902 15285 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240501 08:14:11.251258 15285 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240501 08:14:11.252478 15285 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240501 08:14:11.254417 15285 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240501 08:14:11.254519 15285 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:11.254508 15148 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
I20240501 08:14:11.254652 15285 ts_tablet_manager.cc:590] Registered 0 tablets
I20240501 08:14:11.254701 15285 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240501 08:14:11.289548 15285 rpc_server.cc:307] RPC server started. Bound to: 127.28.1.136:37957
I20240501 08:14:11.289595 15413 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.1.136:37957 every 8 connection(s)
I20240501 08:14:11.291347 15285 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240501 08:14:11.300552 28678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVWMLaH/build/asan/bin/kudu as pid 15285
I20240501 08:14:11.318791 15414 heartbeater.cc:347] Connected to a master server at 127.28.1.190:37815
I20240501 08:14:11.319140 15414 heartbeater.cc:464] Registering TS with master...
I20240501 08:14:11.320210 15414 heartbeater.cc:511] Master 127.28.1.190:37815 requested a full tablet report, sending...
I20240501 08:14:11.321861 12522 ts_manager.cc:194] Re-registered known tserver with Master: 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957)
I20240501 08:14:11.322988 12522 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.1.136:35033
I20240501 08:14:11.756012 15282 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
I20240501 08:14:11.926426 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:11.944120 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:11.965703 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:11.978077 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.016861 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.041180 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.054431 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:12.074647 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.155390 13050 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:12.181298 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.326272 15414 heartbeater.cc:503] Master 127.28.1.190:37815 was elected leader, sending a full tablet report...
I20240501 08:14:12.389991 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.409569 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.411389 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.423805 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.470320 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.503718 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.521725 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.538163 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240501 08:14:12.544034 13138 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 104262us
I20240501 08:14:12.579859 13050 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:12.601491 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.870695 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.871358 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.881947 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.909790 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.914284 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:12.925998 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:12.968462 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:12.990784 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:13.001456 13050 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:13.088953 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:13.169593 13201 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:13.172737 13467 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.172991 13334 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.173345 12938 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.173411 15567 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.173720 15520 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.173871 13069 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.174767 15567 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.175199 15520 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.179248 13069 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.179042 15567 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.184746 12528 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.188403 12938 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.190127 12803 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.190155 12671 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.191099 15520 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.191488 15568 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.194875 13467 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.194873 13334 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.209136 12671 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.211815 12803 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.228521 13201 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:13.233666 12938 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.233701 12671 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.234937 15520 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.235500 15585 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.236430 12803 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.237378 13334 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.239076 15568 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.239156 13069 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.239320 15567 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.239821 15568 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.239950 13467 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.242497 15568 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.243147 15520 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.243937 12803 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.244982 13334 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.248682 13069 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.250702 12531 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.252974 13467 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.253911 12938 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.254946 13070 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:13.259398 12671 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:13.273079 13201 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.273334 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.273981 15469 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.274650 15469 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.276175 13334 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.277743 15468 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.286708 12806 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.287436 15468 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.287892 12671 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.289852 15469 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.304574 12937 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.305197 15468 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.306118 15469 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.306641 13465 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.309262 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 8b196f49443f4479837a06e16e7aca6b with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:13.312085 13335 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.312893 12531 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
W20240501 08:14:13.317072 13136 consensus_peers.cc:485] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b196f49443f4479837a06e16e7aca6b. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:13.320238 13201 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.323943 12937 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.341914 13070 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:13.345651 13335 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.346015 12937 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.347343 15468 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.347577 13202 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.348152 12672 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.348780 15598 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.349354 15598 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.348286 12806 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.350351 12672 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.354571 13465 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.355337 12803 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.356518 15598 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.356882 15599 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.357578 15468 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.362265 15599 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.363010 12803 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.364543 12937 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.364789 13466 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.368763 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 795dd1b66844485dad769acd4ec2b16f with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:13.370471 12529 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.370484 13202 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.371088 13334 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:13.374730 12672 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
W20240501 08:14:13.389351 13003 consensus_peers.cc:485] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 795dd1b66844485dad769acd4ec2b16f. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:13.394155 12672 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:13.397068 13202 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.397078 13465 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.397708 15286 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.397874 13335 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.398648 13070 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.398794 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.399329 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.399755 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.410301 12937 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.410980 12803 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.411021 15608 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240501 08:14:13.412163 15286 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.416527 15286 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.423550 13465 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.422593 13335 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.431020 12531 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.432245 12938 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.435321 13070 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.437085 13202 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.440454 12672 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:13.445791 12806 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.447161 13335 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.452195 13202 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.456566 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.457168 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.458132 12806 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.461068 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.464473 15557 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.465772 12803 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.473520 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:13.474175 12530 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.489331 12938 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.490684 15614 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.492758 12938 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.493896 13070 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.494549 15616 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.494258 13335 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.496657 13070 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.499074 13202 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.516641 13465 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.519297 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.521132 13465 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
W20240501 08:14:13.528419 12609 consensus_peers.cc:485] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2ede6a5ca6654aceb2a24e1592fc1bc8. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:13.548559 13335 consensus_queue.cc:237] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:13.551924 12938 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.551903 12672 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.552778 15420 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.553212 12803 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.553310 15420 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.554181 15153 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.569633 15153 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.570722 12672 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.575130 13070 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.577085 13465 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.577450 15420 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.578895 12529 catalog_manager.cc:5484] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.580416 13202 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.581473 15420 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.592983 15633 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.010s
I20240501 08:14:13.612385 12937 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.616649 12803 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.622318 13070 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.626681 13202 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:13.629570 13466 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
W20240501 08:14:13.647583 13269 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:45349, user_credentials={real_user=slave}} blocked reactor thread for 52126.4us
I20240501 08:14:13.654491 13335 consensus_queue.cc:237] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:13.657672 13202 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.658506 15633 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.659099 12803 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.659902 15153 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.660117 12937 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.660439 13465 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.660454 12672 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.660871 13069 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.661163 15633 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.661413 15153 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.661695 15632 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.662969 15633 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.664898 15637 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.665915 13465 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.666079 12937 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.666606 12672 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.672248 12530 catalog_manager.cc:5484] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.670282 13070 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.674386 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 8a12727f56884a45bcd89631b986eab7 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:13.675415 13202 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.675590 12803 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
W20240501 08:14:13.683348 13271 consensus_peers.cc:485] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8a12727f56884a45bcd89631b986eab7. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:13.689882 13465 consensus_queue.cc:237] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } }
I20240501 08:14:13.693106 12937 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.693341 12672 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.693945 15422 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.694151 12803 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.694325 15647 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.695214 15422 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.696770 13070 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.697540 15519 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.709648 13202 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.715813 15422 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.721376 15422 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.723165 12672 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.724361 13070 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.727914 15653 ts_tablet_manager.cc:902] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:13.729600 12531 catalog_manager.cc:5484] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.731153 13335 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.733731 15653 tablet_copy_client.cc:308] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:13.731766 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.009s
I20240501 08:14:13.742542 12803 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.742956 12937 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.751703 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet 8b196f49443f4479837a06e16e7aca6b from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:46121)
I20240501 08:14:13.752046 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet 8b196f49443f4479837a06e16e7aca6b from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:46121: session id = 35cb9fadf231408598c3877837a6149e-8b196f49443f4479837a06e16e7aca6b
I20240501 08:14:13.753439 13335 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.758093 13465 consensus_queue.cc:237] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:13.763535 15665 ts_tablet_manager.cc:902] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:13.764755 13224 tablet_copy_source_session.cc:205] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:13.767031 13335 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.767401 12803 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.768043 12672 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.768994 13070 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.769941 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.770457 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.770872 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.771839 15653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:13.772400 15649 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.777232 15647 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.779433 12937 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.780860 12672 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.781035 13202 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } }
I20240501 08:14:13.782807 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.784569 12937 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.787623 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3c85628288804001a4777a0f00269836 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:13.791819 13070 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.791858 13334 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.795840 12522 catalog_manager.cc:5484] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
W20240501 08:14:13.798995 13403 consensus_peers.cc:485] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3c85628288804001a4777a0f00269836. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:13.799705 12803 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.811787 13201 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:13.812682 15662 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:13.814395 13201 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:13.832167 15665 tablet_copy_client.cc:308] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:13.846470 15653 tablet_copy_client.cc:783] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:13.846921 15653 tablet_copy_client.cc:647] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:13.850250 15653 tablet_copy_client.cc:523] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:13.854238 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 795dd1b66844485dad769acd4ec2b16f from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:54593)
I20240501 08:14:13.854554 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 795dd1b66844485dad769acd4ec2b16f from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:54593: session id = 8e309f48ce254660915fb9fc32649ab2-795dd1b66844485dad769acd4ec2b16f
I20240501 08:14:13.857883 13090 tablet_copy_source_session.cc:205] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:13.858219 15653 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:13.863663 15665 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:13.867408 15674 ts_tablet_manager.cc:902] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:13.872545 15653 log.cc:824] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Log is configured to *not* fsync() on all Append() calls
I20240501 08:14:13.878233 12937 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:13.881155 13465 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.881172 15665 tablet_copy_client.cc:783] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:13.881440 13335 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.881597 15665 tablet_copy_client.cc:647] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:13.882210 14959 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.883106 13070 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.884472 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.884953 15665 tablet_copy_client.cc:523] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:13.886265 13201 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.888831 15592 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.889680 15665 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:13.891778 15674 tablet_copy_client.cc:308] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:13.930141 15665 log.cc:824] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Log is configured to *not* fsync() on all Append() calls
I20240501 08:14:13.947822 15653 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:13.948541 15653 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:13.949187 15653 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.091s user 0.067s sys 0.000s
I20240501 08:14:13.953716 15596 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.956161 12672 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.956974 12803 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:13.956890 15653 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:13.957692 15653 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:13.957842 14959 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.958242 15684 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.958389 15653 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:13.958850 15683 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:13.962517 13465 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.973676 12525 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.981907 13335 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:13.993721 13202 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:14.001577 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:37487)
I20240501 08:14:14.001845 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:37487: session id = 3cd5ec97de8642dd8796fae63ea73e5a-2ede6a5ca6654aceb2a24e1592fc1bc8
I20240501 08:14:14.005863 12694 tablet_copy_source_session.cc:205] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:13.994295 13070 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:14.011376 15674 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:14.010917 12803 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:14.020421 12937 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.027091 13335 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.028805 13202 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.029543 15653 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.080s user 0.020s sys 0.032s
I20240501 08:14:14.029048 15695 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240501 08:14:14.029368 12672 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:14.029853 14959 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:14.030303 15674 tablet_copy_client.cc:783] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:14.030740 15674 tablet_copy_client.cc:647] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:14.030952 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-8b196f49443f4479837a06e16e7aca6b received from {username='slave'} at 127.28.1.136:46121
I20240501 08:14:14.031118 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session 35cb9fadf231408598c3877837a6149e-8b196f49443f4479837a06e16e7aca6b on tablet 8b196f49443f4479837a06e16e7aca6b with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:14.034219 15674 tablet_copy_client.cc:523] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:14.039072 13465 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.040184 15674 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:14.040603 15695 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:14.043396 14959 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.049603 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 2b098b5bce924e6ea5897e463451b964 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:14.049959 13465 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.050045 13202 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.053467 12525 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:14.060937 13335 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
W20240501 08:14:14.063655 12874 consensus_peers.cc:485] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2b098b5bce924e6ea5897e463451b964. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:14.068035 12671 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.069401 15665 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:14.069998 15665 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:14.070492 15665 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.181s user 0.057s sys 0.010s
I20240501 08:14:14.074043 15683 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:14.075884 12671 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.076759 15674 log.cc:824] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Log is configured to *not* fsync() on all Append() calls
I20240501 08:14:14.077052 15665 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:14.077600 15665 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:14.078203 15665 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:14.087172 13069 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.088116 15665 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.017s user 0.017s sys 0.001s
I20240501 08:14:14.088402 15683 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:14.089027 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-795dd1b66844485dad769acd4ec2b16f received from {username='slave'} at 127.28.1.137:54593
I20240501 08:14:14.089216 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-795dd1b66844485dad769acd4ec2b16f on tablet 795dd1b66844485dad769acd4ec2b16f with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:14.090421 13069 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.096815 12806 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:14.097615 15697 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:14.099352 12806 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:14.146214 15711 ts_tablet_manager.cc:902] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:14.148939 15711 tablet_copy_client.cc:308] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:14.153484 15674 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:14.153878 15718 ts_tablet_manager.cc:902] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:14.154240 15674 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:14.154848 15674 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.116s user 0.064s sys 0.010s
I20240501 08:14:14.162230 15674 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.162974 15674 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:14.163823 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet 8a12727f56884a45bcd89631b986eab7 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:45065)
I20240501 08:14:14.163733 15674 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.164155 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet 8a12727f56884a45bcd89631b986eab7 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:45065: session id = 3cd5ec97de8642dd8796fae63ea73e5a-8a12727f56884a45bcd89631b986eab7
I20240501 08:14:14.167474 13355 tablet_copy_source_session.cc:205] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:14.168819 15674 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.014s user 0.011s sys 0.000s
I20240501 08:14:14.169574 15711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8a12727f56884a45bcd89631b986eab7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:14.169907 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-2ede6a5ca6654aceb2a24e1592fc1bc8 received from {username='slave'} at 127.28.1.138:37487
I20240501 08:14:14.170094 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-2ede6a5ca6654aceb2a24e1592fc1bc8 on tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:14.175007 15711 tablet_copy_client.cc:783] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:14.175269 15711 tablet_copy_client.cc:647] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:14.177863 15711 tablet_copy_client.cc:523] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:14.179477 15718 tablet_copy_client.cc:308] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:14.181555 15711 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:14.187871 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 3c85628288804001a4777a0f00269836 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:48359)
I20240501 08:14:14.188181 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 3c85628288804001a4777a0f00269836 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:48359: session id = 3cd5ec97de8642dd8796fae63ea73e5a-3c85628288804001a4777a0f00269836
I20240501 08:14:14.191814 13488 tablet_copy_source_session.cc:205] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:14.194227 15718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c85628288804001a4777a0f00269836. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:14.200435 15718 tablet_copy_client.cc:783] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:14.200714 15718 tablet_copy_client.cc:647] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:14.204306 15718 tablet_copy_client.cc:523] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:14.210201 15718 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:14.220219 15369 raft_consensus.cc:1203] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.247916 15711 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:14.248446 15711 tablet_bootstrap.cc:492] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:14.249079 15711 ts_tablet_manager.cc:1367] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.068s user 0.046s sys 0.002s
I20240501 08:14:14.250160 15711 raft_consensus.cc:348] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.250491 15711 raft_consensus.cc:729] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:14.250708 15711 consensus_queue.cc:260] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.251334 15711 ts_tablet_manager.cc:1398] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:14.252561 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-8a12727f56884a45bcd89631b986eab7 received from {username='slave'} at 127.28.1.138:45065
I20240501 08:14:14.252758 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-8a12727f56884a45bcd89631b986eab7 on tablet 8a12727f56884a45bcd89631b986eab7 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:14.279224 15718 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:14.279645 15718 tablet_bootstrap.cc:492] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:14.279937 15718 ts_tablet_manager.cc:1367] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.070s user 0.051s sys 0.003s
I20240501 08:14:14.280746 15718 raft_consensus.cc:348] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.281078 15718 raft_consensus.cc:729] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:14.281304 15718 consensus_queue.cc:260] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.281956 15718 ts_tablet_manager.cc:1398] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:14:14.282910 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-3c85628288804001a4777a0f00269836 received from {username='slave'} at 127.28.1.138:48359
I20240501 08:14:14.283105 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-3c85628288804001a4777a0f00269836 on tablet 3c85628288804001a4777a0f00269836 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:14.299496 15103 raft_consensus.cc:1203] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.417384 15718 ts_tablet_manager.cc:902] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:14.418825 15718 tablet_copy_client.cc:308] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:14.424258 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 2b098b5bce924e6ea5897e463451b964 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:36317)
I20240501 08:14:14.424484 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 2b098b5bce924e6ea5897e463451b964 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:36317: session id = 3cd5ec97de8642dd8796fae63ea73e5a-2b098b5bce924e6ea5897e463451b964
I20240501 08:14:14.427045 12959 tablet_copy_source_session.cc:205] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:14.428772 15718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:14.433239 15718 tablet_copy_client.cc:783] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:14.433218 15237 raft_consensus.cc:1203] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.433678 15718 tablet_copy_client.cc:647] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:14.435500 15718 tablet_copy_client.cc:523] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:14.438416 15718 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:14.472972 15718 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:14.473448 15718 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:14.473727 15718 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.035s user 0.036s sys 0.000s
I20240501 08:14:14.474457 15718 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.474752 15718 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:14.474992 15718 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:14.475459 15718 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:14.476327 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-2b098b5bce924e6ea5897e463451b964 received from {username='slave'} at 127.28.1.138:36317
I20240501 08:14:14.476486 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-2b098b5bce924e6ea5897e463451b964 on tablet 2b098b5bce924e6ea5897e463451b964 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:14.587651 15237 raft_consensus.cc:1203] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.608551 15237 raft_consensus.cc:1203] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.823863 15616 raft_consensus.cc:1050] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:14.825188 15616 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:14.826515 15736 raft_consensus.cc:1050] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:14.827664 15736 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:14.828400 15237 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.828936 13335 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.829633 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.829705 13203 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.830607 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.830873 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.836350 13465 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.836400 13070 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.836941 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.838282 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.840199 12803 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.840296 12937 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.840859 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.841277 15616 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.849370 15557 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.851020 12806 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.852429 13069 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.857754 13334 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.858763 12531 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.856413 15237 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.863504 13203 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.884795 15369 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.885814 13465 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.886512 12671 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.886592 13334 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.887560 13070 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.888350 15588 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.888932 15588 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.889459 15588 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.889717 12938 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.894992 15742 raft_consensus.cc:1050] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:14.896229 15742 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:14.898422 13466 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.904359 12671 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:14.904788 15103 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.905490 15605 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.906424 12937 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.908241 12806 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.909230 12937 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.909407 13334 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.909855 15616 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.910221 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.910737 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.913895 13466 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.914839 15237 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.919809 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.920428 13334 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.920470 15557 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.921042 15605 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.923662 15616 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.927796 15237 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.927935 13466 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.934522 12937 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.936054 12530 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.936774 15833 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.938393 15833 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.942514 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:14.942860 13201 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:14.944180 13333 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.944715 12806 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:14.949016 12938 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.949210 12671 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.949891 15835 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.950067 13465 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.950508 15835 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:14.951929 13050 tablet_service.cc:1514] Processing DeleteTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:14.953127 15839 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240501 08:14:14.966068 13139 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:43031, user_credentials={real_user=slave}} blocked reactor thread for 44437.8us
I20240501 08:14:14.967204 13203 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.966908 15828 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240501 08:14:14.969617 15617 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:14.977267 15237 raft_consensus.cc:1203] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:14.984385 12806 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:14.985248 15832 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240501 08:14:14.998670 13005 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 50776us
I20240501 08:14:14.999707 12806 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.001515 15605 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.007872 15637 raft_consensus.cc:1050] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:15.009150 15637 consensus_queue.cc:237] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.027839 15828 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.032449 13333 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.034803 15605 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.040153 13465 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.041482 12530 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.043303 12672 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.050067 15237 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.050662 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.061281 15647 raft_consensus.cc:1050] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:15.062548 15647 consensus_queue.cc:237] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.078337 15843 tablet_replica.cc:330] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:15.079070 15843 raft_consensus.cc:2229] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:15.079371 12805 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.079568 15843 raft_consensus.cc:2258] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:15.080184 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.080257 12937 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.082398 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.085983 15742 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
W20240501 08:14:15.089576 12608 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.133:42529, user_credentials={real_user=slave}} blocked reactor thread for 120755us
I20240501 08:14:15.091678 13465 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.092537 13203 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.092675 12938 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.093205 12527 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:15.100780 12936 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.104838 15103 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.107532 12672 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.108325 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.124279 15843 ts_tablet_manager.cc:1875] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.124477 13334 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.125013 13069 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.131918 12670 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.135757 15237 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.136454 15662 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.143823 13465 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.143826 13069 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.143735 12803 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.144590 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.144937 15632 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.147696 12934 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.148340 13203 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.148978 12670 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.148944 15632 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.150475 12804 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.151132 13334 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.151333 15662 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.151504 13200 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.151907 15662 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.152804 13069 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.152761 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.153303 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.153734 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240501 08:14:15.160205 13136 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.136:37957, user_credentials={real_user=slave}} blocked reactor thread for 271596us
I20240501 08:14:15.163511 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.182302 15632 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.190832 15236 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.193063 12672 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.201304 13070 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.201978 13200 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.204950 12938 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.205354 13202 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.209158 12530 catalog_manager.cc:5484] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.216316 12805 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.221328 12802 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.224467 13070 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:15.228180 15103 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.228801 12671 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.229421 15840 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.228993 15662 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.230325 15840 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.234434 13466 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.235606 15840 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.242518 12936 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.242103 13334 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.241533 15236 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.244374 15369 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.248598 12529 catalog_manager.cc:5484] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.249498 15605 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.249513 13200 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.253242 15742 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.254828 13466 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.256901 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 795dd1b66844485dad769acd4ec2b16f with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:15.258307 13068 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.259413 12670 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.255861 12937 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.261638 13466 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.255864 12936 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.261567 12672 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.265928 13334 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.267056 12805 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.267807 15605 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.268421 15840 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.269182 15103 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.268417 12802 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.273456 12522 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:15.279747 13334 consensus_queue.cc:237] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.281842 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 795dd1b66844485dad769acd4ec2b16f with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:15.284438 12938 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.285046 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.285492 12671 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.286412 12805 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.287405 15632 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.288194 15637 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.290536 15236 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.293200 13467 consensus_queue.cc:237] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.296051 15843 ts_tablet_manager.cc:1888] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.296274 15843 log.cc:1196] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/2ede6a5ca6654aceb2a24e1592fc1bc8
I20240501 08:14:15.297464 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:15.298434 15236 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.298377 12802 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.299060 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.300091 12805 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.300318 12937 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.300431 15874 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.005s
I20240501 08:14:15.302675 13334 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:15.303759 13465 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.303891 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.304126 15637 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.305598 15784 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.305714 15237 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.307087 12671 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.308540 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8a12727f56884a45bcd89631b986eab7 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:15.309298 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.310743 13465 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.312544 12672 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.314613 15648 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240501 08:14:15.318106 13068 raft_consensus.cc:1261] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.317914 12529 catalog_manager.cc:5484] T 8a12727f56884a45bcd89631b986eab7 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.318289 12936 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.320287 15874 consensus_queue.cc:1035] T 8a12727f56884a45bcd89631b986eab7 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.322306 13068 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.324869 13182 tablet_service.cc:1514] Processing DeleteTablet for tablet 8a12727f56884a45bcd89631b986eab7 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:15.326133 15878 tablet_replica.cc:330] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:15.326630 15878 raft_consensus.cc:2229] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:15.327072 15878 raft_consensus.cc:2258] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:15.328212 15648 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.331115 13334 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.309422 12805 raft_consensus.cc:2932] T 8a12727f56884a45bcd89631b986eab7 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.335335 13200 raft_consensus.cc:1261] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.335741 15882 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.345198 15880 consensus_queue.cc:1035] T 3c85628288804001a4777a0f00269836 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.345734 12805 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.347882 13200 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.352463 15878 ts_tablet_manager.cc:1875] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.352511 12936 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.359882 13334 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.363415 12529 catalog_manager.cc:5484] T 3c85628288804001a4777a0f00269836 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.368544 13050 tablet_service.cc:1514] Processing DeleteTablet for tablet 3c85628288804001a4777a0f00269836 with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:15.368963 15843 tablet_replica.cc:330] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:15.369169 15843 raft_consensus.cc:2229] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
W20240501 08:14:15.375576 13003 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.137:37989, user_credentials={real_user=slave}} blocked reactor thread for 46172.7us
I20240501 08:14:15.376031 15843 raft_consensus.cc:2258] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
W20240501 08:14:15.376168 13005 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 48372.7us
I20240501 08:14:15.378319 15843 ts_tablet_manager.cc:1875] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.386495 15843 ts_tablet_manager.cc:1888] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.386726 15843 log.cc:1196] T 3c85628288804001a4777a0f00269836 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/3c85628288804001a4777a0f00269836
I20240501 08:14:15.387528 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 3c85628288804001a4777a0f00269836 (table rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]) successfully deleted
I20240501 08:14:15.389674 15236 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.393657 15683 raft_consensus.cc:1050] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:15.394976 15683 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.395321 15878 ts_tablet_manager.cc:1888] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.397785 15878 log.cc:1196] T 795dd1b66844485dad769acd4ec2b16f P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/795dd1b66844485dad769acd4ec2b16f
W20240501 08:14:15.400923 13401 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:45349, user_credentials={real_user=slave}} blocked reactor thread for 62039.3us
I20240501 08:14:15.401402 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3c85628288804001a4777a0f00269836 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:15.401985 12672 raft_consensus.cc:2932] T 3c85628288804001a4777a0f00269836 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.406790 15878 tablet_replica.cc:330] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:15.407133 15878 raft_consensus.cc:2229] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:15.407778 15878 raft_consensus.cc:2258] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:15.409308 15878 ts_tablet_manager.cc:1875] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.398795 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 795dd1b66844485dad769acd4ec2b16f (table rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]) successfully deleted
I20240501 08:14:15.414216 15878 ts_tablet_manager.cc:1888] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.414342 15878 log.cc:1196] T 8a12727f56884a45bcd89631b986eab7 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/8a12727f56884a45bcd89631b986eab7
I20240501 08:14:15.419545 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 8a12727f56884a45bcd89631b986eab7 (table rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]) successfully deleted
I20240501 08:14:15.423255 15236 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.423854 13334 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.424052 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.424660 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.428449 13200 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.428449 13068 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.428671 13467 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.429173 15697 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.429759 15697 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.430667 12805 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.431167 15697 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.431383 12672 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:15.432514 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.432754 15683 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:15.452955 15683 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.459746 15236 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.464126 12522 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:15.467401 12802 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.467507 13467 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.464556 13334 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.475373 13200 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.459767 13068 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.492679 12672 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.495688 12936 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:15.499290 15236 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.499323 13467 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.499656 13068 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.500137 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.500444 13334 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.500890 15683 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.501719 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.502120 15596 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.505277 12802 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.505406 15596 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.506181 12672 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.506556 15899 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.508383 15697 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.508281 12802 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.510043 13467 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.517362 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2b098b5bce924e6ea5897e463451b964 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:15.518889 12522 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.518990 13070 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.518685 15236 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.522606 12672 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.528856 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 2b098b5bce924e6ea5897e463451b964 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:15.529335 15878 tablet_replica.cc:330] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:15.529548 15878 raft_consensus.cc:2229] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:15.529809 15878 raft_consensus.cc:2258] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:15.531183 15878 ts_tablet_manager.cc:1875] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.543552 13334 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:15.555131 15878 ts_tablet_manager.cc:1888] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.557807 15878 log.cc:1196] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/2b098b5bce924e6ea5897e463451b964
I20240501 08:14:15.558981 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 2b098b5bce924e6ea5897e463451b964 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:15.581297 13200 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:15.584719 13334 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.585068 15369 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.585285 15832 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.586634 12936 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.586733 13467 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.587821 12802 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.588129 15829 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.590107 15829 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.590636 15829 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.592373 12672 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:15.592914 15736 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.594880 15736 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:15.596611 15832 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.598459 12936 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.600708 13467 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.601387 12802 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.601970 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b196f49443f4479837a06e16e7aca6b with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:15.603461 15369 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.603696 13334 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.605158 12672 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:15.608232 12522 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:15.613029 13050 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b196f49443f4479837a06e16e7aca6b with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:15.613477 15843 tablet_replica.cc:330] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:15.613696 15843 raft_consensus.cc:2229] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:15.613958 15843 raft_consensus.cc:2258] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:15.615528 15843 ts_tablet_manager.cc:1875] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:15.626835 15843 ts_tablet_manager.cc:1888] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:15.626993 15843 log.cc:1196] T 8b196f49443f4479837a06e16e7aca6b P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/8b196f49443f4479837a06e16e7aca6b
I20240501 08:14:15.627795 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 8b196f49443f4479837a06e16e7aca6b (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:15.848508 12936 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:15.851560 13334 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.851735 13467 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.852094 13068 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.852722 15683 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.852962 13200 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.853034 15899 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.853313 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.853927 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.856863 12802 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.858018 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.860460 12672 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.860395 15899 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.861079 15683 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.862279 13467 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.870263 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.870683 12529 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.872066 12670 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.877743 13068 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.878772 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.888386 12802 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.890144 12936 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:15.897049 12802 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:15.897089 13201 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:15.897944 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.898430 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.900012 13334 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:15.900524 15596 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.903229 15901 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.904851 13068 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:15.905342 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.906612 13068 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.908249 13467 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:15.908658 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 1226795e4f534366874b23270d086334 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:15.908926 12802 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.909461 15596 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.911078 13467 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.912443 12670 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240501 08:14:15.913026 12871 consensus_peers.cc:485] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1226795e4f534366874b23270d086334. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:15.912386 12527 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.915480 15596 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:15.917430 12670 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.918928 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.921841 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:15.944886 13467 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:15.949163 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.949985 12670 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.950037 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.950764 15885 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.951051 13201 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.951678 15885 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.952515 12802 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.953065 15885 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.956564 12936 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.957340 15885 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.959447 13334 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:15.960121 15885 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:15.983752 15885 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.985812 12670 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.985842 12802 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.986737 12936 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.989442 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.994884 12522 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:15.999369 13333 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.009692 13467 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.009976 13201 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.016741 12936 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.017407 15951 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.020121 12802 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.021978 12670 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.022663 15951 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.025480 13201 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.025558 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.025799 13333 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.026012 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.026937 15954 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.028580 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.029453 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.029896 15662 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.030675 12802 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.033247 13070 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.036456 13200 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.039734 12525 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.043640 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 16288057e4f24e8ebe4fd959d026d9d8 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:16.044265 12670 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.049257 12936 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
W20240501 08:14:16.052803 13400 consensus_peers.cc:485] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 16288057e4f24e8ebe4fd959d026d9d8. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:16.054194 13333 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.073132 12670 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:16.076550 13467 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.076823 13201 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.077169 13333 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.077160 15846 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.077447 15616 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.078122 15960 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.078860 13070 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.079308 12936 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.079423 15616 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.080121 15616 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.104974 12802 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.105815 15961 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.106541 15961 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.107432 13467 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.113124 12529 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:16.115527 12936 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.116624 13201 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.118855 13070 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.125656 13333 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.130100 12806 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.139091 12670 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.142695 12936 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.142925 13333 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.143491 15846 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.143599 13070 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.144068 13467 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.144766 13201 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.145664 15616 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.146152 15846 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.146708 15846 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.147439 15616 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.153123 13201 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.154271 13070 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.156365 12806 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.157073 15962 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.157369 12936 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.158926 12806 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
W20240501 08:14:16.160645 12606 consensus_peers.cc:485] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 70ff1e2f7acd4e0784fef633bd0fd64d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:16.151700 15616 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.163861 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:16.166077 12529 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.167135 12805 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.167740 12937 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.167819 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.168298 15840 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.170572 13467 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.171198 15742 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.172721 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 70ff1e2f7acd4e0784fef633bd0fd64d with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:16.174039 13333 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.176414 12670 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.180166 13466 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:16.188127 15742 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.191129 13201 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.191676 13333 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.192749 15840 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.193292 15742 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.199638 15840 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.203097 13333 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.204309 13201 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.204278 12937 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.205255 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.208700 12670 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.208910 12527 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.213049 12805 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.225332 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
W20240501 08:14:16.228490 13003 consensus_peers.cc:485] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9f65863a31714bd19691d8eb5187ca91. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:16.228543 12670 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.228904 13333 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.229104 15985 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.229342 13466 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.229460 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.231981 15985 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.232967 13201 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.233577 15742 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.235574 15985 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.236964 13201 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.239867 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.244863 12531 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.247159 12937 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.247269 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 9f65863a31714bd19691d8eb5187ca91 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:16.248283 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.249368 13333 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.250195 12937 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.250761 12805 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.250466 12670 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.251339 15742 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.255210 12806 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.261080 13201 consensus_queue.cc:237] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:16.265651 12937 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.267170 15969 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.269707 12806 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.270295 15969 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.265838 12670 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.271596 15969 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.272058 13466 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.272660 13333 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.273579 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.273687 13068 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.274055 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.274293 15969 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.289625 15832 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.298696 12937 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.300735 12531 catalog_manager.cc:5484] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:16.305123 13068 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.306737 13466 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.308799 13200 consensus_queue.cc:237] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.328454 13068 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.329082 15994 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240501 08:14:16.334785 13136 consensus_peers.cc:485] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: baba1d19f76a434b858e635590a50c7c. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:16.338290 13333 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.339278 15994 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.340806 13333 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.345953 12670 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.347412 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.349336 13466 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.348994 12672 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.351500 12937 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.352670 15994 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.354852 15994 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.356750 12805 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.359740 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.360250 15832 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.364681 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet baba1d19f76a434b858e635590a50c7c with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:16.365561 13068 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.367003 12806 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:16.367565 12937 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.368541 13333 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.369819 12525 catalog_manager.cc:5484] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:16.373507 12806 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.374011 12672 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.385444 13466 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.399680 16010 ts_tablet_manager.cc:902] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:16.416774 16010 tablet_copy_client.cc:308] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:16.433615 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 1226795e4f534366874b23270d086334 from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:36601)
I20240501 08:14:16.433811 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 1226795e4f534366874b23270d086334 from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:36601: session id = 35cb9fadf231408598c3877837a6149e-1226795e4f534366874b23270d086334
I20240501 08:14:16.436404 12959 tablet_copy_source_session.cc:205] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:16.437960 16010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:16.445271 16010 tablet_copy_client.cc:783] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:16.445547 16010 tablet_copy_client.cc:647] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:16.447324 16010 tablet_copy_client.cc:523] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:16.450909 16010 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:16.499485 16010 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:16.499917 16010 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:16.500226 16010 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.049s user 0.030s sys 0.016s
I20240501 08:14:16.501192 16010 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.501503 16010 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:16.501698 16010 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.502444 16010 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:16.503136 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-1226795e4f534366874b23270d086334 received from {username='slave'} at 127.28.1.136:36601
I20240501 08:14:16.503262 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session 35cb9fadf231408598c3877837a6149e-1226795e4f534366874b23270d086334 on tablet 1226795e4f534366874b23270d086334 with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:16.565410 16010 ts_tablet_manager.cc:902] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:16.566586 16010 tablet_copy_client.cc:308] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:16.572291 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:50307)
I20240501 08:14:16.572470 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:50307: session id = 35cb9fadf231408598c3877837a6149e-16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:16.574358 13488 tablet_copy_source_session.cc:205] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:16.575569 16010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:16.577682 16048 ts_tablet_manager.cc:902] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:16.579224 16048 tablet_copy_client.cc:308] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:16.581483 16010 tablet_copy_client.cc:783] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:16.581897 16010 tablet_copy_client.cc:647] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:16.583684 16010 tablet_copy_client.cc:523] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:16.586102 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet 70ff1e2f7acd4e0784fef633bd0fd64d from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:60059)
I20240501 08:14:16.586287 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet 70ff1e2f7acd4e0784fef633bd0fd64d from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:60059: session id = 35cb9fadf231408598c3877837a6149e-70ff1e2f7acd4e0784fef633bd0fd64d
I20240501 08:14:16.586500 16010 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:16.588461 12694 tablet_copy_source_session.cc:205] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:16.589797 16048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ff1e2f7acd4e0784fef633bd0fd64d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:16.595046 16048 tablet_copy_client.cc:783] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:16.595350 16048 tablet_copy_client.cc:647] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:16.597515 16048 tablet_copy_client.cc:523] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:16.600703 16048 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:16.631295 16010 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:16.631786 16010 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:16.632093 16010 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.046s user 0.029s sys 0.018s
I20240501 08:14:16.633059 16010 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.633399 16010 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:16.633651 16010 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.634294 16010 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:16.635133 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-16288057e4f24e8ebe4fd959d026d9d8 received from {username='slave'} at 127.28.1.136:50307
I20240501 08:14:16.635270 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 35cb9fadf231408598c3877837a6149e-16288057e4f24e8ebe4fd959d026d9d8 on tablet 16288057e4f24e8ebe4fd959d026d9d8 with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:16.646422 16048 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:16.646800 16048 tablet_bootstrap.cc:492] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:16.647049 16048 ts_tablet_manager.cc:1367] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.046s user 0.042s sys 0.002s
I20240501 08:14:16.647655 16048 raft_consensus.cc:348] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.647897 16048 raft_consensus.cc:729] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:16.648118 16048 consensus_queue.cc:260] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:16.648628 16048 ts_tablet_manager.cc:1398] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.001s user 0.000s sys 0.003s
I20240501 08:14:16.649574 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-70ff1e2f7acd4e0784fef633bd0fd64d received from {username='slave'} at 127.28.1.136:60059
I20240501 08:14:16.649710 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 35cb9fadf231408598c3877837a6149e-70ff1e2f7acd4e0784fef633bd0fd64d on tablet 70ff1e2f7acd4e0784fef633bd0fd64d with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:16.707996 12672 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:16.711401 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.711748 13200 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.711794 13333 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.712074 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.712509 12937 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.713122 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.713316 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.715150 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.715672 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.716812 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.721884 16062 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.722651 13466 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.726056 12937 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.727066 12525 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:16.728842 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.740989 12806 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:16.741921 13333 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.748153 13200 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.753324 12672 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.758334 12806 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.758739 12937 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.759443 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.759617 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.760103 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.760360 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.761587 13333 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.761696 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.761929 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:16.762511 15846 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.763522 13200 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:16.766001 16061 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.767504 12806 raft_consensus.cc:858] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }:
New config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
W20240501 08:14:16.767684 12806 raft_consensus.cc:1565] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 5' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }:
New config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.768733 16063 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:16.769327 12806 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:16.770316 13333 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.770284 12937 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.772384 13200 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.776160 13467 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.777630 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.782989 12527 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.783632 12806 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:16.796553 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet cd1c9882eb034801aa59324fe069461e with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
W20240501 08:14:16.805091 12606 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd1c9882eb034801aa59324fe069461e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:16.805315 16081 ts_tablet_manager.cc:902] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:16.806962 16081 tablet_copy_client.cc:308] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:16.807680 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 9f65863a31714bd19691d8eb5187ca91 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:54593)
I20240501 08:14:16.808916 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 9f65863a31714bd19691d8eb5187ca91 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:54593: session id = 8e309f48ce254660915fb9fc32649ab2-9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:16.814930 13090 tablet_copy_source_session.cc:205] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:16.815382 16086 ts_tablet_manager.cc:902] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:16.816474 16081 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:16.821887 16081 tablet_copy_client.cc:783] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:16.822214 16081 tablet_copy_client.cc:647] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:16.824612 16081 tablet_copy_client.cc:523] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:16.828344 16081 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:16.853835 16086 tablet_copy_client.cc:308] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:16.875614 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet baba1d19f76a434b858e635590a50c7c from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:43669)
I20240501 08:14:16.875810 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet baba1d19f76a434b858e635590a50c7c from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:43669: session id = 8e309f48ce254660915fb9fc32649ab2-baba1d19f76a434b858e635590a50c7c
I20240501 08:14:16.879244 13224 tablet_copy_source_session.cc:205] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:16.880956 16086 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet baba1d19f76a434b858e635590a50c7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:16.888440 16086 tablet_copy_client.cc:783] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:16.888715 16086 tablet_copy_client.cc:647] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:16.889662 16081 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:16.890113 16081 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:16.890614 16081 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.062s user 0.055s sys 0.000s
I20240501 08:14:16.891691 16081 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.892184 16081 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:16.892436 16086 tablet_copy_client.cc:523] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:16.892499 16081 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.893579 16081 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:14:16.894762 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-9f65863a31714bd19691d8eb5187ca91 received from {username='slave'} at 127.28.1.137:54593
I20240501 08:14:16.894896 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-9f65863a31714bd19691d8eb5187ca91 on tablet 9f65863a31714bd19691d8eb5187ca91 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:16.898075 16086 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:16.942301 16086 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:16.942695 16086 tablet_bootstrap.cc:492] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:16.942998 16086 ts_tablet_manager.cc:1367] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.045s user 0.034s sys 0.004s
I20240501 08:14:16.943702 16086 raft_consensus.cc:348] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.944077 16086 raft_consensus.cc:729] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:16.944317 16086 consensus_queue.cc:260] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:16.944885 16086 ts_tablet_manager.cc:1398] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:14:16.945669 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-baba1d19f76a434b858e635590a50c7c received from {username='slave'} at 127.28.1.137:43669
I20240501 08:14:16.945787 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-baba1d19f76a434b858e635590a50c7c on tablet baba1d19f76a434b858e635590a50c7c with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:16.970229 15369 raft_consensus.cc:1203] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.046514 15369 raft_consensus.cc:1203] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.065066 15369 raft_consensus.cc:1203] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.239331 15103 raft_consensus.cc:1203] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.241176 16086 ts_tablet_manager.cc:902] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:17.242931 16086 tablet_copy_client.cc:308] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:17.249715 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet cd1c9882eb034801aa59324fe069461e from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:52199)
I20240501 08:14:17.249892 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet cd1c9882eb034801aa59324fe069461e from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:52199: session id = 8e309f48ce254660915fb9fc32649ab2-cd1c9882eb034801aa59324fe069461e
I20240501 08:14:17.251891 12694 tablet_copy_source_session.cc:205] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:17.253748 16086 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:17.259490 16086 tablet_copy_client.cc:783] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:17.259742 16086 tablet_copy_client.cc:647] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:17.261907 16086 tablet_copy_client.cc:523] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:17.264564 16086 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:17.299320 16086 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:17.299832 16086 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:17.300149 16086 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.036s user 0.035s sys 0.000s
I20240501 08:14:17.301021 16086 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:17.301362 16086 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:17.301676 15103 raft_consensus.cc:1203] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.301607 16086 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:17.306126 16086 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.006s user 0.001s sys 0.004s
I20240501 08:14:17.306962 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-cd1c9882eb034801aa59324fe069461e received from {username='slave'} at 127.28.1.137:52199
I20240501 08:14:17.307096 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-cd1c9882eb034801aa59324fe069461e on tablet cd1c9882eb034801aa59324fe069461e with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:17.456768 16116 raft_consensus.cc:1050] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:17.458021 16116 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.462309 13334 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.463486 15369 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.463553 16115 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.464283 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.464546 13068 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.464578 13467 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.464764 13200 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.466200 16115 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.466509 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.467098 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.487618 12670 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.487648 12805 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.488386 16182 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.488770 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.496760 15697 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.497537 12670 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.507457 12527 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.522424 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.522449 13200 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.522460 15369 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.526738 12805 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.535377 13070 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.544289 13467 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.553558 12937 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.556450 13334 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.557106 15369 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.557901 16194 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.558241 12670 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.559954 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.560755 13467 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.561333 15697 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.561527 13201 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.565268 16181 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.565670 16194 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.566233 16194 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.566763 12670 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.571182 15369 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.574636 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1226795e4f534366874b23270d086334 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:17.575860 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.576089 13467 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.581076 12530 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.584057 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.588158 12805 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.590868 16182 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.593029 12805 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.602072 13050 tablet_service.cc:1514] Processing DeleteTablet for tablet 1226795e4f534366874b23270d086334 with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:17.602720 16200 tablet_replica.cc:330] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:17.602941 16200 raft_consensus.cc:2229] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:17.603315 16200 raft_consensus.cc:2258] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:17.604406 16200 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:17.610152 16200 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:17.610271 16200 log.cc:1196] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/1226795e4f534366874b23270d086334
I20240501 08:14:17.610950 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 1226795e4f534366874b23270d086334 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:17.622768 15662 raft_consensus.cc:1050] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:17.623697 15662 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.626514 13334 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.626600 13201 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.626967 13070 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.627236 16117 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.627513 15369 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.627650 12805 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.627717 16117 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.628063 16201 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.628562 16201 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.628747 12670 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.628796 12937 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.629392 16203 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.629693 16201 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.630224 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.648963 16203 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.649618 13070 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.650458 13200 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.654629 13334 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.656414 12528 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.658393 12806 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.660391 15369 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.664132 12670 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
W20240501 08:14:17.667272 15662 consensus_queue.cc:1159] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "35cb9fadf231408598c3877837a6149e" responder_term: 1 status { last_received { term: 1 index: 6 } last_committed_idx: 6 last_received_current_leader { term: 1 index: 6 } }
I20240501 08:14:17.668195 16062 raft_consensus.cc:1050] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:17.668228 13467 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.669140 16062 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.671926 12936 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.672489 13334 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.672927 13070 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.673288 16201 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.673588 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.673941 16203 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.673143 12937 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.676378 12805 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.677510 16201 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.678202 12670 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.678702 16203 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.678961 15369 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.679705 15662 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.681741 16117 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.682999 12672 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.683945 15368 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.687708 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 16288057e4f24e8ebe4fd959d026d9d8 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:17.688306 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.688980 13333 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.689224 12529 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:17.691224 12805 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.697135 12936 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.700085 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:17.703812 13200 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.704209 12805 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.704244 15368 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.704700 12936 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.704838 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.705185 16062 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.705480 16061 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.707023 15992 raft_consensus.cc:1050] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:17.707358 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.708215 15992 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:17.708770 13068 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.708835 13334 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.709247 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.709725 16061 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.710606 13466 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.711225 16061 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.724870 15103 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.725540 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.732189 16212 tablet_replica.cc:330] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:17.732250 16061 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.733711 12937 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.734256 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.733963 13068 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.735918 12805 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.741019 12670 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.742983 16212 raft_consensus.cc:2229] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:17.743721 16212 raft_consensus.cc:2258] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:17.746052 12530 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:17.746166 15368 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.745316 16212 ts_tablet_manager.cc:1875] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:17.752758 13466 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.756522 15832 raft_consensus.cc:1050] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:17.757073 13467 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.757776 15832 consensus_queue.cc:237] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:17.762580 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.026s
I20240501 08:14:17.763805 13200 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.764019 12805 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.767712 16212 ts_tablet_manager.cc:1888] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:17.769634 16212 log.cc:1196] T 16288057e4f24e8ebe4fd959d026d9d8 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:17.770896 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 16288057e4f24e8ebe4fd959d026d9d8 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:17.775323 15103 raft_consensus.cc:1203] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:17.782620 12936 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.799917 12938 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.800487 13466 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.800804 12805 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.801689 12670 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.801884 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.802492 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.802959 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.803443 13333 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240501 08:14:17.805719 16227 consensus_queue.cc:1159] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "5bf85222f4734b7885eb866efd27d5f2" responder_term: 1 status { last_received { term: 1 index: 6 } last_committed_idx: 6 last_received_current_leader { term: 1 index: 6 } }
I20240501 08:14:17.805806 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.806183 15103 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.806123 12672 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:17.806306 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.809923 15368 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.810184 13467 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.810714 12937 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.812211 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.812729 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.816798 16061 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.819236 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.820353 16061 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.822772 15368 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.835292 13200 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.837098 12806 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.837096 16229 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.837702 16227 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.798365 13334 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.838591 12529 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.839727 13200 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.845400 13335 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.845840 12806 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.852185 16063 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.864593 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 70ff1e2f7acd4e0784fef633bd0fd64d with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:17.867388 13467 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.873397 12938 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.882817 13335 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:17.894685 15832 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.895354 13467 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.909185 15103 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.915256 12529 catalog_manager.cc:5484] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:17.920461 12670 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.929277 15992 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.004s
I20240501 08:14:17.938246 16223 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240501 08:14:17.940024 13005 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 202385us
I20240501 08:14:17.941052 13333 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.942595 13200 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240501 08:14:17.943647 13006 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:43031, user_credentials={real_user=slave}} blocked reactor thread for 186012us
I20240501 08:14:17.950012 13068 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:17.952926 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.954052 15605 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:17.955193 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240501 08:14:17.959517 13138 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 59438.4us
I20240501 08:14:17.960736 12806 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.963286 13333 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
W20240501 08:14:17.967401 13139 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:43031, user_credentials={real_user=slave}} blocked reactor thread for 67014.8us
I20240501 08:14:17.968250 12936 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.968719 13201 consensus_queue.cc:237] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:17.972781 15103 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.973546 13467 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.974439 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.975841 15832 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.976979 12806 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.977533 16005 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.979089 13335 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:17.981806 16005 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.986364 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet baba1d19f76a434b858e635590a50c7c with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:17.987519 16242 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:17.988745 12806 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
W20240501 08:14:17.992333 13004 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:45349, user_credentials={real_user=slave}} blocked reactor thread for 250641us
I20240501 08:14:17.994560 13335 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:17.994968 12530 catalog_manager.cc:5484] T baba1d19f76a434b858e635590a50c7c P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.000033 12670 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.000684 12936 raft_consensus.cc:1261] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.000798 16242 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.002346 16242 consensus_queue.cc:1035] T baba1d19f76a434b858e635590a50c7c P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.004158 12938 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.011440 13050 tablet_service.cc:1514] Processing DeleteTablet for tablet 70ff1e2f7acd4e0784fef633bd0fd64d with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:18.011924 16200 tablet_replica.cc:330] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:18.012169 16200 raft_consensus.cc:2229] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:18.012635 16200 raft_consensus.cc:2258] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:18.011446 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet baba1d19f76a434b858e635590a50c7c with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:18.012701 12670 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.014294 16200 ts_tablet_manager.cc:1875] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:18.022037 16200 ts_tablet_manager.cc:1888] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:18.026446 12805 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.024946 15605 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.027370 16200 log.cc:1196] T 70ff1e2f7acd4e0784fef633bd0fd64d P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/70ff1e2f7acd4e0784fef633bd0fd64d
I20240501 08:14:18.032179 15103 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.032601 12530 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.032539 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.032609 12938 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.039947 16200 tablet_replica.cc:330] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:18.040249 16200 raft_consensus.cc:2229] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:18.040654 16200 raft_consensus.cc:2258] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:18.041468 16200 ts_tablet_manager.cc:1875] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:18.043455 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 70ff1e2f7acd4e0784fef633bd0fd64d (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:18.067318 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.068398 16200 ts_tablet_manager.cc:1888] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240501 08:14:18.068547 16200 log.cc:1196] T baba1d19f76a434b858e635590a50c7c P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/baba1d19f76a434b858e635590a50c7c
I20240501 08:14:18.072559 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet baba1d19f76a434b858e635590a50c7c (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:18.075593 13466 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.077337 15103 raft_consensus.cc:2932] T baba1d19f76a434b858e635590a50c7c P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.091814 13200 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.108907 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:18.112740 12672 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.113328 13466 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.113517 16255 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.114435 15992 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.115360 12938 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.114945 12672 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.118422 12805 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.119202 13335 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.120066 15992 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.003s
I20240501 08:14:18.128971 16225 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.008s
I20240501 08:14:18.129609 16255 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.131433 15992 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.133817 12670 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.138630 12938 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.140949 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9f65863a31714bd19691d8eb5187ca91 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:18.141953 12805 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.142714 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.143774 12530 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.148471 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 9f65863a31714bd19691d8eb5187ca91 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:18.148905 16212 tablet_replica.cc:330] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:18.149113 16212 raft_consensus.cc:2229] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:18.149479 16212 raft_consensus.cc:2258] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:18.150841 16212 ts_tablet_manager.cc:1875] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:18.152213 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.158006 16212 ts_tablet_manager.cc:1888] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:18.158183 16212 log.cc:1196] T 9f65863a31714bd19691d8eb5187ca91 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:18.159320 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 9f65863a31714bd19691d8eb5187ca91 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:18.160655 15103 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.162190 15992 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.164106 15103 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.173413 16062 raft_consensus.cc:1050] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:18.174566 16062 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:18.177714 15103 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.178293 13335 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.178553 16229 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.179015 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.179759 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.179817 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.180356 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.180804 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.185662 13200 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.186188 12805 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.186336 12938 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:18.186817 16062 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.187193 16228 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.187935 16230 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.190069 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.191099 13200 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.193053 15103 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.193642 16061 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:18.196796 13335 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.197831 12522 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.199172 13466 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.210809 12938 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.224365 12805 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.487102 13068 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:18.489953 12670 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.490319 12938 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.490667 13466 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.490937 13200 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.490901 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.491205 15992 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.491510 12805 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.492110 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.492673 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.493878 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.496527 13335 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.497881 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.501785 15605 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.502403 13335 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.508241 13201 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.510143 12938 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.513417 12530 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.513393 12805 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.521298 12670 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.521270 13466 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.529491 13068 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:18.532745 13466 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.533738 16226 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.533941 13335 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.535490 16226 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.537726 13201 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.538385 12805 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.538479 12938 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.538848 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.539441 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.539750 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.542398 12670 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.542428 16223 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.543406 16255 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.545852 12938 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.549732 13200 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.554608 13335 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.557312 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet e9b18b849c914fa5ba731892b7b88d6e with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:18.560340 12522 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.559099 12670 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.564749 12805 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.573352 13466 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
W20240501 08:14:18.574477 13003 consensus_peers.cc:485] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e9b18b849c914fa5ba731892b7b88d6e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:18.601768 13200 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:18.604981 13068 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.605180 12670 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.605767 13335 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.605787 16251 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.606396 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.606760 12938 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.607329 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.608350 16245 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.609356 13466 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.610287 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.623127 12805 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.622980 16005 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.627295 16251 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.627125 13334 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.627178 13070 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.629323 12670 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.631078 12938 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.633370 12530 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.634256 13466 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.640475 13200 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
W20240501 08:14:18.644852 13136 consensus_peers.cc:485] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:18.645484 12670 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.645933 13334 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.646493 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.646747 16245 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.647372 13466 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.651989 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.652978 13068 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.653797 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.655473 12805 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.657387 16245 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.658785 12936 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.661634 13467 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.665138 16251 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.666240 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:18.666868 12938 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.667006 13335 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.667141 12670 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.671358 13070 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.671583 12529 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:18.674134 12806 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.675029 16244 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.676571 12806 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.687624 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:18.690580 12806 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.690713 12938 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.691116 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.691632 12670 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.691937 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.693267 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.699276 13200 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.700099 13466 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.700269 13070 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.700690 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.701068 16325 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.701963 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.707695 15637 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.708192 13466 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.710896 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.713099 12805 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.715147 12938 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.716364 12670 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.719205 12528 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.723073 13200 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:18.736626 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:18.740819 13200 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.741511 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.741468 12670 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.742604 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.745330 12938 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.745414 12805 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.745982 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.746555 13466 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.746582 16325 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.747143 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.748946 13070 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.749668 16238 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.750697 16238 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.753247 13467 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.756466 12805 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.759850 13200 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.761631 12530 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.764024 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet cab182a0e126421c986101ea19df3e97 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:18.765549 12938 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.767659 12672 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:18.769549 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
W20240501 08:14:18.785706 13268 consensus_peers.cc:485] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cab182a0e126421c986101ea19df3e97. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:18.806382 13467 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:18.809937 13200 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.810386 13335 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.810662 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.811167 16203 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.812050 12672 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:18.812453 12938 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.812453 13070 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.813184 16203 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.813661 16203 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.821795 13334 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.822305 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.822554 12805 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.823266 16229 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.823844 16229 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.825201 12936 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.826105 16062 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.829309 16230 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.830863 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.833671 16062 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.838348 16229 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.838646 13070 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.841501 12936 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.845796 13466 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.853102 12522 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.858397 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet cd1c9882eb034801aa59324fe069461e with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:18.859588 16350 tablet_replica.cc:330] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:18.859797 16350 raft_consensus.cc:2229] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:18.859815 15103 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:18.860244 16350 raft_consensus.cc:2258] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:18.860769 16347 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:18.861202 16350 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:18.862540 15103 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.863942 13334 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.865598 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd1c9882eb034801aa59324fe069461e with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:18.866135 15662 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.870572 16350 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:18.870839 16350 log.cc:1196] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/cd1c9882eb034801aa59324fe069461e
I20240501 08:14:18.870630 12805 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:18.870905 12936 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.873919 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet cd1c9882eb034801aa59324fe069461e (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:18.883869 12672 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.884608 16203 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.884647 12522 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.886309 12805 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:18.883527 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.890863 16203 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:18.893399 13200 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.895931 13070 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.905454 12805 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.919080 13466 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:18.917457 12672 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:18.923171 13335 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.923946 12805 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.925084 16353 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.925621 16353 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240501 08:14:18.926970 13400 consensus_peers.cc:485] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 85eb7c44993b4749b4b0a090d773bbef. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:18.926777 12936 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.933872 16353 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.939613 16353 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.945312 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 85eb7c44993b4749b4b0a090d773bbef with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:18.946777 12936 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.950238 13068 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.951432 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240501 08:14:18.953523 12671 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.955353 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.957047 12671 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.957413 13070 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.961742 12522 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.964267 13200 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:18.965013 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:18.966797 13200 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.968770 12805 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:18.968981 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.035041 16369 ts_tablet_manager.cc:902] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:19.047338 16369 tablet_copy_client.cc:308] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:19.057047 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet e9b18b849c914fa5ba731892b7b88d6e from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:54165)
I20240501 08:14:19.057235 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet e9b18b849c914fa5ba731892b7b88d6e from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:54165: session id = 35cb9fadf231408598c3877837a6149e-e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:19.059854 13090 tablet_copy_source_session.cc:205] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.061661 16369 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.067149 16369 tablet_copy_client.cc:783] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.067420 16369 tablet_copy_client.cc:647] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.071791 16369 tablet_copy_client.cc:523] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.075186 16369 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:19.129256 16369 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:19.129894 16369 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:19.130573 16369 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.055s user 0.046s sys 0.008s
I20240501 08:14:19.131719 16369 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:19.132076 16369 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:19.132364 16369 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:19.134490 16369 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240501 08:14:19.136929 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-e9b18b849c914fa5ba731892b7b88d6e received from {username='slave'} at 127.28.1.136:54165
I20240501 08:14:19.137113 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 35cb9fadf231408598c3877837a6149e-e9b18b849c914fa5ba731892b7b88d6e on tablet e9b18b849c914fa5ba731892b7b88d6e with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:19.258548 16398 ts_tablet_manager.cc:902] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:19.260362 16398 tablet_copy_client.cc:308] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:19.261127 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:43669)
I20240501 08:14:19.261309 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:43669: session id = 8e309f48ce254660915fb9fc32649ab2-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:19.263414 13224 tablet_copy_source_session.cc:205] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.264767 16398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.270208 16398 tablet_copy_client.cc:783] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.270484 16398 tablet_copy_client.cc:647] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.272122 16398 tablet_copy_client.cc:523] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.276335 16398 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:19.311002 16398 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:19.311369 16398 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:19.311617 16398 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.035s user 0.032s sys 0.000s
I20240501 08:14:19.312300 16398 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:19.312592 16398 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:19.312790 16398 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:19.313273 16398 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:14:19.314154 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b received from {username='slave'} at 127.28.1.137:43669
I20240501 08:14:19.314299 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:19.346689 13466 consensus_queue.cc:237] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } }
I20240501 08:14:19.349531 13200 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.349499 13070 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.349900 12671 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.350062 13334 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.350200 12936 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.350800 16353 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.351168 16362 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.351421 12806 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.351473 16356 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.351739 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.352007 16203 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.352628 15662 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.358170 16356 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.359004 12806 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.359009 13334 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.363579 13070 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.364384 12938 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.364820 12529 catalog_manager.cc:5484] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.365746 16398 ts_tablet_manager.cc:902] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:19.367843 12671 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.387490 13070 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:19.390801 12938 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.391419 13466 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.391469 12805 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.391599 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.392181 16226 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.394093 12671 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.395354 13201 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.395992 16225 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.396504 13334 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:19.397552 16225 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.402386 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.402725 16226 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.399657 13467 consensus_queue.cc:237] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:19.406240 12934 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.406791 16356 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.406980 13333 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.407742 12806 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240501 08:14:19.407887 13403 consensus_peers.cc:485] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7ac049d157cd4cc495c747c02be17bca. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:19.408069 16356 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.408627 12672 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.410241 13202 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.412873 16356 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.413301 16356 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.413425 13068 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.416656 13202 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } }
I20240501 08:14:19.418963 16356 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.420936 16226 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.422518 12806 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.425370 12934 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.428732 15662 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.430241 15662 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.431825 12936 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.432241 13332 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.437882 12531 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } }
I20240501 08:14:19.441238 12529 catalog_manager.cc:5484] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.447645 13466 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.460018 13202 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.459877 13201 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.472483 12805 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.485152 13070 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
W20240501 08:14:19.488148 13401 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.129:45349, user_credentials={real_user=slave}} blocked reactor thread for 57952.1us
I20240501 08:14:19.490404 13202 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240501 08:14:19.490906 13003 consensus_peers.cc:485] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e8782ed1d2a5492e99b9bf6bd4b1597a. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:19.491518 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.493163 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 7ac049d157cd4cc495c747c02be17bca with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:19.494412 13068 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.497959 16369 ts_tablet_manager.cc:902] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:19.502213 12672 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:19.503569 13467 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.504179 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.509639 12671 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.509721 12934 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.510294 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.510689 16225 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.513116 12671 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.514256 16369 tablet_copy_client.cc:308] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:19.516258 12806 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.517671 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 85eb7c44993b4749b4b0a090d773bbef from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:50307)
I20240501 08:14:19.517843 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 85eb7c44993b4749b4b0a090d773bbef from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:50307: session id = 35cb9fadf231408598c3877837a6149e-85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:19.518464 16225 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:19.520812 13488 tablet_copy_source_session.cc:205] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.522450 16369 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.526098 16225 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.526804 12806 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.530261 16369 tablet_copy_client.cc:783] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.530596 16369 tablet_copy_client.cc:647] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.534735 13201 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.535001 12527 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.537766 13334 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:19.542529 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:19.543999 13335 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:19.544708 16369 tablet_copy_client.cc:523] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.553471 13335 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.558368 12516 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet e8782ed1d2a5492e99b9bf6bd4b1597a with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:19.560134 13467 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.564970 12934 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.567585 16369 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:19.588352 12671 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:19.595062 16398 tablet_copy_client.cc:308] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:19.636502 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet cab182a0e126421c986101ea19df3e97 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:39889)
I20240501 08:14:19.636687 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet cab182a0e126421c986101ea19df3e97 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:39889: session id = 8e309f48ce254660915fb9fc32649ab2-cab182a0e126421c986101ea19df3e97
I20240501 08:14:19.638713 13355 tablet_copy_source_session.cc:205] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.644421 15368 raft_consensus.cc:1203] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:19.646380 16398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.657891 16398 tablet_copy_client.cc:783] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.658157 16398 tablet_copy_client.cc:647] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.661252 16398 tablet_copy_client.cc:523] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.664853 16398 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:19.678274 16369 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:19.678830 16369 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:19.679216 16369 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.112s user 0.046s sys 0.005s
I20240501 08:14:19.680187 16369 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:19.680626 16369 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:19.680923 16369 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:19.681819 16369 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:19.682859 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-85eb7c44993b4749b4b0a090d773bbef received from {username='slave'} at 127.28.1.136:50307
I20240501 08:14:19.683012 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 35cb9fadf231408598c3877837a6149e-85eb7c44993b4749b4b0a090d773bbef on tablet 85eb7c44993b4749b4b0a090d773bbef with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:19.720181 16398 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:19.720686 16398 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:19.720930 16398 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.056s user 0.036s sys 0.011s
I20240501 08:14:19.721668 16398 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:19.721890 16398 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:19.722097 16398 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:19.722664 16398 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:19.723511 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-cab182a0e126421c986101ea19df3e97 received from {username='slave'} at 127.28.1.137:39889
I20240501 08:14:19.723640 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-cab182a0e126421c986101ea19df3e97 on tablet cab182a0e126421c986101ea19df3e97 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:19.728334 15103 raft_consensus.cc:1203] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:19.832841 16475 ts_tablet_manager.cc:902] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:19.838567 16475 tablet_copy_client.cc:308] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:19.839282 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 7ac049d157cd4cc495c747c02be17bca from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:48359)
I20240501 08:14:19.839471 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 7ac049d157cd4cc495c747c02be17bca from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:48359: session id = 3cd5ec97de8642dd8796fae63ea73e5a-7ac049d157cd4cc495c747c02be17bca
I20240501 08:14:19.840587 15103 raft_consensus.cc:1203] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:19.841944 13488 tablet_copy_source_session.cc:205] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.843286 16475 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ac049d157cd4cc495c747c02be17bca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.848838 16475 tablet_copy_client.cc:783] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.849083 16475 tablet_copy_client.cc:647] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.850577 16475 tablet_copy_client.cc:523] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.853116 16475 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:19.886793 16475 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:19.887146 16475 tablet_bootstrap.cc:492] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:19.887378 16475 ts_tablet_manager.cc:1367] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.034s user 0.032s sys 0.000s
I20240501 08:14:19.887949 16475 raft_consensus.cc:348] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:19.888231 16475 raft_consensus.cc:729] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:19.888382 16475 consensus_queue.cc:260] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:19.889072 16475 ts_tablet_manager.cc:1398] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.001s sys 0.004s
I20240501 08:14:19.889827 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-7ac049d157cd4cc495c747c02be17bca received from {username='slave'} at 127.28.1.138:48359
I20240501 08:14:19.889953 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-7ac049d157cd4cc495c747c02be17bca on tablet 7ac049d157cd4cc495c747c02be17bca with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:19.935456 16369 ts_tablet_manager.cc:902] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:19.952817 16369 tablet_copy_client.cc:308] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:19.953845 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet e8782ed1d2a5492e99b9bf6bd4b1597a from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:54165)
I20240501 08:14:19.954063 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet e8782ed1d2a5492e99b9bf6bd4b1597a from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:54165: session id = 35cb9fadf231408598c3877837a6149e-e8782ed1d2a5492e99b9bf6bd4b1597a
I20240501 08:14:19.956774 13090 tablet_copy_source_session.cc:205] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:19.958760 16369 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:19.966025 16369 tablet_copy_client.cc:783] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:19.966293 16369 tablet_copy_client.cc:647] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:19.967921 16369 tablet_copy_client.cc:523] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:19.971015 16369 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:20.022779 16369 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:20.023231 16369 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:20.023536 16369 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.053s user 0.040s sys 0.008s
I20240501 08:14:20.024322 16369 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:20.024636 16369 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:20.024852 16369 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:20.025523 16369 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240501 08:14:20.027189 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-e8782ed1d2a5492e99b9bf6bd4b1597a received from {username='slave'} at 127.28.1.136:54165
I20240501 08:14:20.027321 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 35cb9fadf231408598c3877837a6149e-e8782ed1d2a5492e99b9bf6bd4b1597a on tablet e8782ed1d2a5492e99b9bf6bd4b1597a with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:20.069967 15368 raft_consensus.cc:1203] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:20.134893 16226 raft_consensus.cc:1050] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:20.136134 16226 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:20.139173 12934 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.139690 12806 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.139670 13201 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.139719 13334 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.140322 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.140605 13465 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.140851 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.140911 15368 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.141093 16519 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.141538 16520 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.141849 16226 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.142704 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.143437 12671 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.144037 16225 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.163892 16225 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.164870 13201 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.169075 13465 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.169572 15368 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.169873 12529 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.173195 12934 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.179594 12671 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.184044 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.188228 13334 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.200186 13070 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:20.202575 15368 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.202564 12934 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.202736 12671 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.203224 16520 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.203595 12806 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.203574 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.203995 16226 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.204303 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.204664 13465 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.205279 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.206346 13334 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.207022 16226 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.208196 16226 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.208979 15368 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.212450 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e9b18b849c914fa5ba731892b7b88d6e with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:20.214203 13465 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.213924 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.215238 12527 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.215725 12934 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.218864 13334 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.222930 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet e9b18b849c914fa5ba731892b7b88d6e with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:20.225564 16532 tablet_replica.cc:330] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:20.225808 16532 raft_consensus.cc:2229] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:20.226198 16532 raft_consensus.cc:2258] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:20.226500 12671 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.227730 16532 ts_tablet_manager.cc:1875] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:20.234001 16532 ts_tablet_manager.cc:1888] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:20.234128 16532 log.cc:1196] T e9b18b849c914fa5ba731892b7b88d6e P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:20.236227 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet e9b18b849c914fa5ba731892b7b88d6e (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:20.290299 16244 raft_consensus.cc:1050] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:20.291270 16244 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:20.297955 15103 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.299119 13334 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.299810 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.301995 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.307622 12934 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.307868 13465 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.308308 13070 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.308790 12671 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.308820 16564 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.309110 16567 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.309454 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.310122 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.311017 12806 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.312495 16567 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.321763 16005 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.323100 12671 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.324342 12806 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.324803 13070 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.328665 15103 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.328758 13465 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.330327 12528 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.333010 13334 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.336555 12934 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.344658 13201 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:20.347700 15368 raft_consensus.cc:1203] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:20.348212 12934 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.348836 12805 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.349354 13465 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.349766 16572 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.349910 13335 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.351065 15103 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.351644 12671 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.350482 16568 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240501 08:14:20.352130 16564 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.352854 16572 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.353116 16005 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.354557 16572 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.356271 16005 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.357101 13465 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.358666 15103 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.359386 12671 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.360884 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:20.363392 12805 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.363713 12936 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.365234 12531 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:20.367524 13335 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.371861 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:20.373175 16480 raft_consensus.cc:1050] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:20.374395 16480 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:20.388284 12936 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.388490 12805 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.388859 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.389443 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.389854 15103 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.393127 16579 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.393587 16575 tablet_replica.cc:330] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:20.393886 16575 raft_consensus.cc:2229] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:20.395085 16575 raft_consensus.cc:2258] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:20.396842 16575 ts_tablet_manager.cc:1875] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:20.399556 12671 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.401633 13070 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.401886 13465 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.402639 16579 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.405009 16579 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.407280 16575 ts_tablet_manager.cc:1888] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:20.407492 16575 log.cc:1196] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:20.410171 15236 raft_consensus.cc:1203] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:20.410346 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:20.417667 16579 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.421531 13465 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.424265 16579 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.427466 12531 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
W20240501 08:14:20.433545 13270 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 44563.2us
I20240501 08:14:20.434299 13201 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.436576 12805 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.441563 15637 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.443698 13201 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.445642 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.450150 12936 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.450719 12671 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.454972 15103 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.461292 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:20.464362 12805 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.464421 13465 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.465092 12671 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.464990 16587 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.465556 16586 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.466012 13070 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.466535 16577 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.467038 16577 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.468470 15103 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.468892 12936 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.469599 16480 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.470824 16480 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.472414 16480 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.473654 15102 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.474632 12934 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.476359 12805 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.478629 12671 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.483314 12522 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.488842 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cab182a0e126421c986101ea19df3e97 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:20.489560 13465 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.489569 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:20.491891 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet cab182a0e126421c986101ea19df3e97 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:20.492274 16532 tablet_replica.cc:330] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:20.492498 16532 raft_consensus.cc:2229] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:20.492735 16532 raft_consensus.cc:2258] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:20.493621 16532 ts_tablet_manager.cc:1875] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:20.500380 16532 ts_tablet_manager.cc:1888] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:20.500627 16532 log.cc:1196] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/cab182a0e126421c986101ea19df3e97
I20240501 08:14:20.501451 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet cab182a0e126421c986101ea19df3e97 (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:20.515642 13465 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:20.521193 12934 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.521334 12671 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.521864 15662 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.522073 12805 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.522540 13335 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.522552 13201 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.523108 13070 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:20.523667 15662 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.523928 16344 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.524492 16344 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.524886 16344 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.525206 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.549140 16344 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.550405 12672 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.552634 16598 raft_consensus.cc:1050] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:20.553804 16598 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:20.556531 12522 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.558537 15368 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.558581 13202 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.559190 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.559686 16424 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.560858 13334 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.560642 13335 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.562184 16424 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.557327 13070 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.569377 12934 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.579159 13465 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:20.583200 13200 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.582813 12805 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.583602 13201 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:20.584745 12671 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.589113 16599 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:20.589408 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:20.590241 12934 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.590765 12806 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.592828 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:20.593334 15662 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:20.593793 12936 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.596233 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.597488 12806 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.594874 13069 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.597972 16344 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.598287 12671 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.599076 16424 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.599574 16424 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.606424 13335 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.606061 16599 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.608599 15662 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:20.610528 12805 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.611402 12938 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.618007 12522 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.619529 13335 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.625521 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 389d0cce047e48d7b702cb7245f60659 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:20.627220 12671 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.628629 13201 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.634718 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:20.636474 16598 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:20.637424 16344 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.638604 13070 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:20.643138 12672 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.650085 12522 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.650211 15368 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.653832 12936 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.660140 13070 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.668802 13201 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.672992 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.677214 12806 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.680379 13465 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:20.683267 15368 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.683674 12938 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.683928 12671 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.684350 15662 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.684633 16424 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.685189 16344 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.688660 13201 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.688592 16598 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.689654 13335 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.690235 16599 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.691161 16599 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.692446 12805 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.692776 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 85eb7c44993b4749b4b0a090d773bbef with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:20.692891 13201 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.693718 15368 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.693954 12938 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.696709 12671 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.698130 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.701195 12531 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.704073 16599 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240501 08:14:20.708024 13400 consensus_peers.cc:485] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 389d0cce047e48d7b702cb7245f60659. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:20.709280 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet 85eb7c44993b4749b4b0a090d773bbef with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:20.709650 16575 tablet_replica.cc:330] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:20.709864 16575 raft_consensus.cc:2229] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:20.710100 16575 raft_consensus.cc:2258] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:20.710063 12806 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.711109 16575 ts_tablet_manager.cc:1875] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:20.716553 16575 ts_tablet_manager.cc:1888] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:20.716758 16575 log.cc:1196] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:20.717872 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 85eb7c44993b4749b4b0a090d773bbef (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:20.825610 16226 raft_consensus.cc:1050] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:20.826491 16226 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:20.832340 12806 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.832432 15369 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.832926 12938 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.832950 16520 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.834509 13465 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.834640 12671 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.835417 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.836350 16647 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240501 08:14:20.837266 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.837812 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.844062 16598 raft_consensus.cc:1050] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:20.845297 16598 consensus_queue.cc:237] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:20.858186 16647 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.858882 12938 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.867027 15236 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.866652 12522 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.867424 13335 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.867687 16606 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.868767 13201 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.869074 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.870769 12671 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.871395 13335 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.871939 12806 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.873142 15369 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.875313 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.877002 13201 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.877707 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.877776 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.878938 13467 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.880015 12938 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.880635 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.905316 13070 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.907299 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.911855 12671 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.912144 13201 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.912747 16344 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.913336 12806 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:20.914670 16599 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:20.918951 13335 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:20.924252 16344 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.930539 12671 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.931099 12936 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.935165 13068 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.936005 13335 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.938127 12528 catalog_manager.cc:5484] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:20.930559 13201 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.945986 15236 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.954888 12806 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.961377 13467 consensus_queue.cc:237] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:20.963948 15236 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.964370 12936 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.964748 16668 ts_tablet_manager.cc:902] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:20.965056 16424 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.965167 13201 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.965626 16424 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.965981 13335 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.966153 16424 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.966487 16599 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.967198 12806 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.968480 16599 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.969493 12671 raft_consensus.cc:1261] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:20.970484 16599 consensus_queue.cc:1035] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:20.972985 16598 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.973397 12671 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.974714 13202 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.977058 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7ac049d157cd4cc495c747c02be17bca with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:20.978240 12936 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.980705 15236 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.983054 12530 catalog_manager.cc:5484] T 7ac049d157cd4cc495c747c02be17bca P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.988046 12805 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:20.991262 13332 raft_consensus.cc:2932] T 7ac049d157cd4cc495c747c02be17bca P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:21.006500 16668 tablet_copy_client.cc:308] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:21.009236 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet 7ac049d157cd4cc495c747c02be17bca with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:21.009639 16575 tablet_replica.cc:330] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:21.009836 16575 raft_consensus.cc:2229] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:21.010229 16575 raft_consensus.cc:2258] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:21.011669 16575 ts_tablet_manager.cc:1875] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:21.014310 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 389d0cce047e48d7b702cb7245f60659 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:45251)
I20240501 08:14:21.014446 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 389d0cce047e48d7b702cb7245f60659 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:45251: session id = 8e309f48ce254660915fb9fc32649ab2-389d0cce047e48d7b702cb7245f60659
I20240501 08:14:21.016412 16575 ts_tablet_manager.cc:1888] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:21.016489 13488 tablet_copy_source_session.cc:205] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:21.016597 16575 log.cc:1196] T 7ac049d157cd4cc495c747c02be17bca P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/7ac049d157cd4cc495c747c02be17bca
I20240501 08:14:21.017545 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet 7ac049d157cd4cc495c747c02be17bca (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:21.018136 16668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:21.022433 16668 tablet_copy_client.cc:783] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:21.022897 16668 tablet_copy_client.cc:647] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:21.024632 16668 tablet_copy_client.cc:523] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:21.027173 16668 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:21.032286 13332 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:21.035828 12936 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.036039 13467 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.036633 16682 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.036815 13070 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.036815 12803 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.037344 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.037343 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.038018 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.038475 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.038939 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.039362 12671 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.043505 16682 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.061278 16480 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.067443 12803 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.072988 12531 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.075708 12672 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.076982 13467 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.084272 16668 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:21.084699 16668 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:21.085003 16668 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.058s user 0.049s sys 0.000s
I20240501 08:14:21.085788 16668 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:21.086103 16668 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:21.086316 16668 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:21.090503 13332 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:21.092103 16668 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.007s user 0.007s sys 0.001s
I20240501 08:14:21.094657 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-389d0cce047e48d7b702cb7245f60659 received from {username='slave'} at 127.28.1.137:45251
I20240501 08:14:21.094811 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-389d0cce047e48d7b702cb7245f60659 on tablet 389d0cce047e48d7b702cb7245f60659 with peer 8e309f48ce254660915fb9fc32649ab2
W20240501 08:14:21.100167 13268 consensus_peers.cc:485] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b6828130a760491eb21fd3d346219748. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:21.100333 12803 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.101275 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.106351 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.107426 16480 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.108736 13202 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.112110 13070 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.114125 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.117234 12671 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.117380 12936 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.117942 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.118077 13467 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.118228 16577 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.118183 13070 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.118763 16577 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.120950 16577 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.125439 12672 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.126385 12517 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet b6828130a760491eb21fd3d346219748 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:21.127532 13466 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.129995 13070 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.131556 12529 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:21.133975 12803 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.136523 12938 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.137128 13202 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.143517 12938 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:21.160275 12938 consensus_queue.cc:237] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } }
I20240501 08:14:21.163446 12672 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.163424 13467 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.163486 12803 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.164289 16690 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.164650 16699 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.164995 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.173465 13202 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.174145 13332 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.174566 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.175267 16701 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.176023 13070 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240501 08:14:21.177261 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.002s
I20240501 08:14:21.179212 16690 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.180044 13202 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.180591 13332 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.183295 12672 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.184988 13466 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.188470 12527 catalog_manager.cc:5484] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.193099 12803 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.203358 12938 consensus_queue.cc:237] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:21.206161 13202 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.206315 13332 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240501 08:14:21.206591 12874 consensus_peers.cc:485] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c3cbbfb0c743496fb4fddfed91bd6913. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:21.206938 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.207372 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.208211 13467 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.208405 12672 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.209049 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.209590 13070 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.209589 16701 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.210892 12803 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240501 08:14:21.211746 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240501 08:14:21.213131 13070 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } }
I20240501 08:14:21.214433 16701 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240501 08:14:21.214725 16700 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.217038 12672 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.219810 12518 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet c3cbbfb0c743496fb4fddfed91bd6913 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240501 08:14:21.220176 13467 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.222242 12531 catalog_manager.cc:5484] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:21.224572 12803 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.225529 13202 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.230001 13332 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.246240 13070 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:21.292732 13070 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:21.295727 12938 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.295917 12803 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.296226 13332 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.297045 16647 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.297185 13467 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.297267 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.297446 16226 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.297955 15369 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.298461 12672 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:21.298738 16647 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.299638 16650 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.300010 16223 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:21.303790 16647 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.305553 13467 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.307322 15369 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.307432 12672 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.309334 12938 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.312661 12522 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.315322 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e8782ed1d2a5492e99b9bf6bd4b1597a with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:21.316740 13332 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.317380 12803 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:21.320152 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:21.329160 16735 tablet_replica.cc:330] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:21.329660 16735 raft_consensus.cc:2229] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:21.330016 16735 raft_consensus.cc:2258] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:21.331437 16735 ts_tablet_manager.cc:1875] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:21.336735 16735 ts_tablet_manager.cc:1888] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:21.336948 16735 log.cc:1196] T e8782ed1d2a5492e99b9bf6bd4b1597a P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/e8782ed1d2a5492e99b9bf6bd4b1597a
I20240501 08:14:21.337700 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet e8782ed1d2a5492e99b9bf6bd4b1597a (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:21.468353 15103 raft_consensus.cc:1203] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:21.514011 16668 ts_tablet_manager.cc:902] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:21.515705 16668 tablet_copy_client.cc:308] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:21.516479 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet b6828130a760491eb21fd3d346219748 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:39889)
I20240501 08:14:21.516655 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet b6828130a760491eb21fd3d346219748 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:39889: session id = 8e309f48ce254660915fb9fc32649ab2-b6828130a760491eb21fd3d346219748
I20240501 08:14:21.519644 13355 tablet_copy_source_session.cc:205] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:21.521196 16668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:21.528415 16668 tablet_copy_client.cc:783] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:21.528689 16668 tablet_copy_client.cc:647] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:21.530736 16668 tablet_copy_client.cc:523] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:21.534257 16668 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:21.578255 16668 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:21.578722 16668 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:21.579025 16668 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.045s user 0.033s sys 0.009s
I20240501 08:14:21.579854 16668 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:21.580191 16668 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:21.580438 16668 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:21.581050 16668 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240501 08:14:21.581732 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-b6828130a760491eb21fd3d346219748 received from {username='slave'} at 127.28.1.137:39889
I20240501 08:14:21.581862 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-b6828130a760491eb21fd3d346219748 on tablet b6828130a760491eb21fd3d346219748 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:21.768435 16756 ts_tablet_manager.cc:902] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:21.771157 16756 tablet_copy_client.cc:308] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:21.771797 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet c3cbbfb0c743496fb4fddfed91bd6913 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:36317)
I20240501 08:14:21.771934 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet c3cbbfb0c743496fb4fddfed91bd6913 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:36317: session id = 3cd5ec97de8642dd8796fae63ea73e5a-c3cbbfb0c743496fb4fddfed91bd6913
I20240501 08:14:21.773885 12959 tablet_copy_source_session.cc:205] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:21.775147 16756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3cbbfb0c743496fb4fddfed91bd6913. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:21.780270 16756 tablet_copy_client.cc:783] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:21.780553 16756 tablet_copy_client.cc:647] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:21.782176 16756 tablet_copy_client.cc:523] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:21.785750 16756 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:21.829906 16756 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:21.830416 16756 tablet_bootstrap.cc:492] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:21.830729 16756 ts_tablet_manager.cc:1367] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.045s user 0.035s sys 0.009s
I20240501 08:14:21.831532 16756 raft_consensus.cc:348] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:21.831881 16756 raft_consensus.cc:729] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:21.832129 16756 consensus_queue.cc:260] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:21.836522 16756 ts_tablet_manager.cc:1398] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.006s user 0.003s sys 0.003s
I20240501 08:14:21.837318 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-c3cbbfb0c743496fb4fddfed91bd6913 received from {username='slave'} at 127.28.1.138:36317
I20240501 08:14:21.837457 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-c3cbbfb0c743496fb4fddfed91bd6913 on tablet c3cbbfb0c743496fb4fddfed91bd6913 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:21.941534 16599 raft_consensus.cc:1050] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:21.942586 16599 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:21.945858 12803 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.945858 15103 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.946106 12938 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.946564 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.947090 16598 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.947726 16598 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.958623 12672 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.958940 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.959233 13332 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.959558 16599 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.960211 16599 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.960686 16599 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.967990 13202 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:21.968783 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:21.971454 16424 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.972514 12672 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.977321 15103 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.977180 12938 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.978018 13202 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.978888 12803 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.982792 12522 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.986126 13068 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:21.990567 13334 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.019471 13467 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:22.022017 15102 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.022222 12938 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.022612 12803 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.022727 16793 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.023084 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.023561 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.023658 13334 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.023833 12671 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.024080 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.024334 16598 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.024778 16424 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.025874 16599 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.028425 16424 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.029093 13068 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.030261 12803 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.031061 12671 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.034039 13334 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.034209 15102 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.034216 12938 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.035672 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 389d0cce047e48d7b702cb7245f60659 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:22.037209 12525 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 6 to 7, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:22.046562 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 389d0cce047e48d7b702cb7245f60659 with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:22.047252 16796 tablet_replica.cc:330] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:22.047454 16796 raft_consensus.cc:2229] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:22.047708 16796 raft_consensus.cc:2258] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:22.048617 16796 ts_tablet_manager.cc:1875] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:22.055310 16796 ts_tablet_manager.cc:1888] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:22.055554 16796 log.cc:1196] T 389d0cce047e48d7b702cb7245f60659 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/389d0cce047e48d7b702cb7245f60659
I20240501 08:14:22.057618 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 389d0cce047e48d7b702cb7245f60659 (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:22.057997 15102 raft_consensus.cc:1203] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:22.312197 15236 raft_consensus.cc:1203] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240501 08:14:22.648396 16692 raft_consensus.cc:1050] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:22.649603 16692 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:22.652644 12938 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.652838 12803 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.652875 13202 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.653649 15102 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.653834 16577 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.655136 16577 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240501 08:14:22.655752 16837 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.656436 16837 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.669886 13068 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.670279 12672 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.670496 16835 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.674222 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.684980 13467 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.685694 16837 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.692222 16836 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.693029 15102 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.694116 12938 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.697367 12803 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.697685 13202 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.699601 12529 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:22.704313 13467 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.703429 13068 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.714124 12672 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.720690 13334 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:22.723865 12938 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.724599 13202 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.725157 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.725751 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.727033 12672 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.728587 13467 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.728686 16835 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.729192 16835 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.732092 16835 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.732865 12671 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.736227 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b6828130a760491eb21fd3d346219748 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:22.737401 12938 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.739104 12531 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.740449 15103 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.741458 12806 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.742064 16836 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.744165 15103 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.746320 13467 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.749712 13202 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.752058 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.753877 12806 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:22.761896 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet b6828130a760491eb21fd3d346219748 with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:22.772477 16856 tablet_replica.cc:330] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:22.772691 16856 raft_consensus.cc:2229] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:22.773224 16856 raft_consensus.cc:2258] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:22.774155 16856 ts_tablet_manager.cc:1875] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:22.779448 16856 ts_tablet_manager.cc:1888] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:22.779683 16856 log.cc:1196] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/b6828130a760491eb21fd3d346219748
I20240501 08:14:22.780467 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet b6828130a760491eb21fd3d346219748 (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:22.870545 16700 raft_consensus.cc:1050] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:22.871417 16700 consensus_queue.cc:237] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:22.874898 12806 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.874898 15236 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.874928 13202 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.875645 16864 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.876384 13334 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.877662 16866 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.878206 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.878499 16866 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.882400 12671 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.883790 13467 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.885619 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.887260 13068 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240501 08:14:22.887866 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.895443 16864 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240501 08:14:22.905841 16864 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.911783 15236 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.912820 12525 catalog_manager.cc:5484] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:22.913076 13467 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.918042 12806 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.920380 12672 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.924646 12938 consensus_queue.cc:237] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:22.926903 13202 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.930562 13466 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.910635 13334 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.931304 16865 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.931699 12803 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.933241 16194 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.932300 13068 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { replace: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.934605 15236 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.935288 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.938151 16865 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.940105 13201 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.939700 15236 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.941172 13332 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.941859 12672 raft_consensus.cc:1261] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240501 08:14:22.943181 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.943996 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.944917 16700 consensus_queue.cc:1035] T c3cbbfb0c743496fb4fddfed91bd6913 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240501 08:14:22.948101 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c3cbbfb0c743496fb4fddfed91bd6913 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:22.947820 13332 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.949638 12806 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.951123 12529 catalog_manager.cc:5484] T c3cbbfb0c743496fb4fddfed91bd6913 P 3cd5ec97de8642dd8796fae63ea73e5a reported cstate change: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.952750 13467 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.959517 13201 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.959821 12672 raft_consensus.cc:2932] T c3cbbfb0c743496fb4fddfed91bd6913 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:22.968168 13049 tablet_service.cc:1514] Processing DeleteTablet for tablet c3cbbfb0c743496fb4fddfed91bd6913 with delete_type TABLET_DATA_TOMBSTONED (TS f59d8b4fd79049b6b40dcecd13d0be9d not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:45936
I20240501 08:14:22.968602 16856 tablet_replica.cc:330] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: stopping tablet replica
I20240501 08:14:22.968839 16856 raft_consensus.cc:2229] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:22.969226 16856 raft_consensus.cc:2258] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:22.970181 16856 ts_tablet_manager.cc:1875] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:22.974968 16856 ts_tablet_manager.cc:1888] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240501 08:14:22.975060 16856 log.cc:1196] T c3cbbfb0c743496fb4fddfed91bd6913 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-3/wal/wals/c3cbbfb0c743496fb4fddfed91bd6913
I20240501 08:14:22.975775 12516 catalog_manager.cc:4830] TS f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): tablet c3cbbfb0c743496fb4fddfed91bd6913 (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:23.343392 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.379125 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.398419 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.402820 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.403043 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.457289 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:23.503731 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.536036 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.541041 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:23.558493 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.773228 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.774080 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.802502 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.809001 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.813060 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.850260 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:23.896966 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.899866 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:23.921821 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:23.949846 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:24.064978 13467 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
W20240501 08:14:24.068964 13403 consensus_peers.cc:485] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 389d0cce047e48d7b702cb7245f60659. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.069793 15103 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.070290 12936 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.070493 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.070966 12806 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.071128 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.071748 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.072055 13332 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.075037 16841 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.087327 16964 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.088017 13332 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.093173 12530 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.099598 12806 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.104405 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.107517 12672 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.109953 16984 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.110832 16841 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.120545 12936 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.123059 12672 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.124344 15103 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.130338 13068 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.153771 13332 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.157624 12936 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.158285 12806 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.158411 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.159183 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.169099 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.169112 13467 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.169721 16993 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.170132 12671 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.170229 16994 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.171516 16692 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.178941 16994 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.180662 12806 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.183239 15103 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.185828 13467 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.188105 12530 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.194275 12936 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.196902 17001 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:24.197574 13268 consensus_peers.cc:485] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b6828130a760491eb21fd3d346219748. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.199120 12670 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.222793 13201 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.224380 15103 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.236728 13068 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:24.240231 15369 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240501 08:14:24.240391 13003 consensus_peers.cc:485] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e9b18b849c914fa5ba731892b7b88d6e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.240814 16780 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.241876 12936 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.242455 16780 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.257592 12806 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.257674 12670 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.257896 13467 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.258508 13332 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.258728 17010 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.259189 16780 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.259716 16780 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.260639 16223 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.273622 16223 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.277652 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.278873 13467 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.282552 12672 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.287642 12530 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.287593 12936 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.287837 15369 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.294143 13332 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.314594 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
W20240501 08:14:24.318722 13003 consensus_peers.cc:485] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9f65863a31714bd19691d8eb5187ca91. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.318801 15103 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.319016 12806 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.320770 13332 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.320914 12936 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.322429 13467 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.323103 16780 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.323621 17018 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.324183 16223 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.324719 16223 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.325228 17026 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.330626 12672 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.331323 17013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.333789 12938 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.338215 12672 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.340925 13467 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.341918 12803 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.344058 12525 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.344996 16780 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.351948 15103 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.353471 13332 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.369753 13467 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:24.373004 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.373023 12672 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.373368 15369 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.373891 12803 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.373859 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.374212 13332 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240501 08:14:24.374771 13403 consensus_peers.cc:485] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 16288057e4f24e8ebe4fd959d026d9d8. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.375097 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.375377 16841 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.375461 12938 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.375957 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.376250 16841 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.379916 16841 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.408090 16841 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.416666 12938 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.417879 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.418776 13334 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.419485 12525 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:24.427374 15368 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.428921 12672 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.446416 12803 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.472683 13201 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
W20240501 08:14:24.477757 13136 consensus_peers.cc:485] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b196f49443f4479837a06e16e7aca6b. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.478529 15369 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.479171 16661 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.479379 12938 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.479422 13332 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.481386 16661 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.482064 16661 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.486678 16661 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.487933 15369 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.492096 13467 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.493063 17053 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.493719 12672 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.495026 12803 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.494772 12530 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.496371 17057 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.496651 13332 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.498445 12803 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.499789 12938 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.504870 13467 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.506845 17057 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.001s
I20240501 08:14:24.511468 12672 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:24.536612 13201 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.539867 15103 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.540367 17068 ts_tablet_manager.cc:902] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
W20240501 08:14:24.540607 13136 consensus_peers.cc:485] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.540791 17057 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.542991 12938 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.543027 13332 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.543576 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.544292 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.550189 13467 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.550196 12672 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.550709 17057 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.550977 12803 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.551203 17058 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.553597 17058 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.572628 17073 ts_tablet_manager.cc:902] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:24.577654 17058 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.580418 13332 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.586567 12672 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.586585 12803 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.587704 12530 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:24.595908 12938 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.594735 13467 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.599505 17068 tablet_copy_client.cc:308] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:24.600303 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 389d0cce047e48d7b702cb7245f60659 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:48359)
I20240501 08:14:24.600478 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 389d0cce047e48d7b702cb7245f60659 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:48359: session id = 3cd5ec97de8642dd8796fae63ea73e5a-389d0cce047e48d7b702cb7245f60659
I20240501 08:14:24.603188 13488 tablet_copy_source_session.cc:205] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.605573 17068 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 389d0cce047e48d7b702cb7245f60659. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.601133 15103 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:24.624497 17068 tablet_copy_client.cc:783] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:24.624788 17068 tablet_copy_client.cc:647] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:24.626993 17068 tablet_copy_client.cc:523] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:24.628361 17073 tablet_copy_client.cc:308] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:24.630940 17068 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:24.651173 13467 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:24.661924 17091 ts_tablet_manager.cc:902] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:24.665148 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet b6828130a760491eb21fd3d346219748 from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:46921)
I20240501 08:14:24.665357 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet b6828130a760491eb21fd3d346219748 from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:46921: session id = 35cb9fadf231408598c3877837a6149e-b6828130a760491eb21fd3d346219748
I20240501 08:14:24.667069 12672 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.667697 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.670114 15369 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.670746 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.671371 12938 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.671984 13201 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.672566 13332 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.674273 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:24.674557 13403 consensus_peers.cc:485] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 85eb7c44993b4749b4b0a090d773bbef. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:24.675712 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.678025 13355 tablet_copy_source_session.cc:205] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.679493 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.680119 17073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.686159 17068 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:24.686589 17068 tablet_bootstrap.cc:492] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:24.686889 17068 ts_tablet_manager.cc:1367] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.056s user 0.048s sys 0.007s
I20240501 08:14:24.686945 17073 tablet_copy_client.cc:783] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:24.687173 17073 tablet_copy_client.cc:647] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:24.687695 17068 raft_consensus.cc:348] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:24.688063 17068 raft_consensus.cc:729] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:24.688328 17068 consensus_queue.cc:260] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:24.689355 17073 tablet_copy_client.cc:523] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:24.693264 17073 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:24.696655 17068 ts_tablet_manager.cc:1398] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.010s user 0.009s sys 0.000s
I20240501 08:14:24.697924 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-389d0cce047e48d7b702cb7245f60659 received from {username='slave'} at 127.28.1.138:48359
I20240501 08:14:24.698079 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-389d0cce047e48d7b702cb7245f60659 on tablet 389d0cce047e48d7b702cb7245f60659 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:24.707327 12803 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.708106 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.714365 17100 ts_tablet_manager.cc:902] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:24.721750 16990 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.726884 15369 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.734334 12938 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.736555 17100 tablet_copy_client.cc:308] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:24.738478 12529 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:24.740538 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 9f65863a31714bd19691d8eb5187ca91 from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:54165)
I20240501 08:14:24.740742 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 9f65863a31714bd19691d8eb5187ca91 from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:54165: session id = 35cb9fadf231408598c3877837a6149e-9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:24.743737 13090 tablet_copy_source_session.cc:205] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.723826 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.751487 13201 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.753554 17100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.759935 12672 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.767094 17100 tablet_copy_client.cc:783] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:24.767403 17100 tablet_copy_client.cc:647] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:24.769435 17100 tablet_copy_client.cc:523] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:24.778240 17100 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:24.799394 12803 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.827845 17073 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:24.836256 17073 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:24.836643 17073 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.143s user 0.050s sys 0.006s
I20240501 08:14:24.837514 17073 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.837873 17073 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:24.838093 17073 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.838742 17073 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240501 08:14:24.838497 13068 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:24.840435 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-b6828130a760491eb21fd3d346219748 received from {username='slave'} at 127.28.1.136:46921
I20240501 08:14:24.840592 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session 35cb9fadf231408598c3877837a6149e-b6828130a760491eb21fd3d346219748 on tablet b6828130a760491eb21fd3d346219748 with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:24.843050 17068 ts_tablet_manager.cc:902] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:24.856783 15103 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.857635 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.860128 17091 tablet_copy_client.cc:308] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:24.862162 13335 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.862399 12672 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.863016 16780 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.863349 12803 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:24.863385 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.864215 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet e9b18b849c914fa5ba731892b7b88d6e from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:54593)
I20240501 08:14:24.864383 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet e9b18b849c914fa5ba731892b7b88d6e from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:54593: session id = 8e309f48ce254660915fb9fc32649ab2-e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:24.867370 13090 tablet_copy_source_session.cc:205] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.871089 16780 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:24.871809 17091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.874622 17068 tablet_copy_client.cc:308] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:24.875371 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:48359)
I20240501 08:14:24.875543 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:48359: session id = 3cd5ec97de8642dd8796fae63ea73e5a-16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:24.878644 13488 tablet_copy_source_session.cc:205] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.880271 17068 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.888533 17091 tablet_copy_client.cc:783] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:24.888868 17091 tablet_copy_client.cc:647] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:24.891021 17091 tablet_copy_client.cc:523] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:24.891266 17068 tablet_copy_client.cc:783] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:24.891551 17068 tablet_copy_client.cc:647] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:24.893519 17068 tablet_copy_client.cc:523] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:24.900359 17100 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:24.901870 17068 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:24.902354 17100 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:24.903164 17100 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.130s user 0.044s sys 0.012s
I20240501 08:14:24.912179 17100 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.912734 17100 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:24.913012 17100 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:24.914759 17100 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240501 08:14:24.915486 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-9f65863a31714bd19691d8eb5187ca91 received from {username='slave'} at 127.28.1.136:54165
I20240501 08:14:24.915702 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 35cb9fadf231408598c3877837a6149e-9f65863a31714bd19691d8eb5187ca91 on tablet 9f65863a31714bd19691d8eb5187ca91 with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:24.919617 17091 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:24.920135 17128 ts_tablet_manager.cc:902] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:24.936249 16780 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.939647 12806 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.952361 12529 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.961496 17128 tablet_copy_client.cc:308] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:24.962436 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet 8b196f49443f4479837a06e16e7aca6b from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:43669)
I20240501 08:14:24.962677 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet 8b196f49443f4479837a06e16e7aca6b from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:43669: session id = 8e309f48ce254660915fb9fc32649ab2-8b196f49443f4479837a06e16e7aca6b
I20240501 08:14:24.965811 13224 tablet_copy_source_session.cc:205] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:24.969012 12670 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:24.984344 17128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b196f49443f4479837a06e16e7aca6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:24.997826 15103 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:25.016453 12938 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.017441 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.020689 16780 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.022589 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.026091 17100 ts_tablet_manager.cc:902] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:25.034824 17128 tablet_copy_client.cc:783] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:25.035161 17128 tablet_copy_client.cc:647] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:25.042364 17128 tablet_copy_client.cc:523] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
W20240501 08:14:25.063225 13006 consensus_peers.cc:485] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 795dd1b66844485dad769acd4ec2b16f. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:25.065040 17100 tablet_copy_client.cc:308] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:25.065253 17128 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:25.065850 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:46121)
I20240501 08:14:25.066032 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:46121: session id = 35cb9fadf231408598c3877837a6149e-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:25.067777 12938 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:25.069329 13224 tablet_copy_source_session.cc:205] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:25.070875 17100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:25.079694 17100 tablet_copy_client.cc:783] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:25.079939 17100 tablet_copy_client.cc:647] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:25.079910 13335 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:25.086055 17100 tablet_copy_client.cc:523] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:25.089682 17100 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:25.077291 13467 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:25.116636 12672 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:25.118520 15237 raft_consensus.cc:1203] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.132911 15103 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240501 08:14:25.133311 12606 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd1c9882eb034801aa59324fe069461e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:25.133791 16965 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.133973 13467 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.134469 16965 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.136616 17150 ts_tablet_manager.cc:902] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:25.136837 13335 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.137528 16965 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.142009 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.143850 16965 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.146236 16666 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.152323 12803 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.152994 16965 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.155160 12938 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:25.156425 17160 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:25.156167 13466 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.170027 12529 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.172412 12803 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.177067 12938 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.182950 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.193310 13335 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.195216 15103 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:25.216207 17100 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.216747 17100 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:25.217048 17100 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.128s user 0.063s sys 0.000s
I20240501 08:14:25.218034 17100 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:25.218400 17100 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:25.218639 17100 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:25.220469 17100 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240501 08:14:25.221408 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b received from {username='slave'} at 127.28.1.136:46121
I20240501 08:14:25.221575 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session 35cb9fadf231408598c3877837a6149e-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:25.225926 17068 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.226438 17068 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:25.226738 17068 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.325s user 0.074s sys 0.097s
I20240501 08:14:25.227711 17068 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.228060 17068 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:25.228452 15369 raft_consensus.cc:1203] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.228319 17068 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.231778 17068 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240501 08:14:25.235158 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-16288057e4f24e8ebe4fd959d026d9d8 received from {username='slave'} at 127.28.1.138:48359
I20240501 08:14:25.235340 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-16288057e4f24e8ebe4fd959d026d9d8 on tablet 16288057e4f24e8ebe4fd959d026d9d8 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:25.241943 17150 tablet_copy_client.cc:308] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:25.242879 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 85eb7c44993b4749b4b0a090d773bbef from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:48359)
I20240501 08:14:25.243077 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 85eb7c44993b4749b4b0a090d773bbef from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:48359: session id = 3cd5ec97de8642dd8796fae63ea73e5a-85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:25.246105 13488 tablet_copy_source_session.cc:205] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:25.248345 17150 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:25.251765 15236 raft_consensus.cc:1203] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.256291 17150 tablet_copy_client.cc:783] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:25.256562 17150 tablet_copy_client.cc:647] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:25.258467 17150 tablet_copy_client.cc:523] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:25.262148 17150 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:25.293156 17091 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.294508 15369 raft_consensus.cc:1203] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.316282 17091 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:25.316618 17150 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.316656 17091 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.397s user 0.052s sys 0.000s
I20240501 08:14:25.317041 17150 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:25.317320 17150 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.055s user 0.049s sys 0.003s
I20240501 08:14:25.317497 17091 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:25.317857 17091 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:25.318063 17150 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.318113 17091 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:25.318379 17150 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:25.318571 17150 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.318828 17091 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:14:25.319285 17150 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240501 08:14:25.321228 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-85eb7c44993b4749b4b0a090d773bbef received from {username='slave'} at 127.28.1.138:48359
I20240501 08:14:25.321362 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-85eb7c44993b4749b4b0a090d773bbef on tablet 85eb7c44993b4749b4b0a090d773bbef with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:25.323580 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-e9b18b849c914fa5ba731892b7b88d6e received from {username='slave'} at 127.28.1.137:54593
I20240501 08:14:25.323724 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-e9b18b849c914fa5ba731892b7b88d6e on tablet e9b18b849c914fa5ba731892b7b88d6e with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:25.335265 17128 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.335767 17128 tablet_bootstrap.cc:492] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:25.336107 17128 ts_tablet_manager.cc:1367] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.271s user 0.052s sys 0.003s
I20240501 08:14:25.336956 17128 raft_consensus.cc:348] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:25.337316 17128 raft_consensus.cc:729] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:25.337544 17128 consensus_queue.cc:260] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:25.338269 17128 ts_tablet_manager.cc:1398] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240501 08:14:25.339493 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-8b196f49443f4479837a06e16e7aca6b received from {username='slave'} at 127.28.1.137:43669
I20240501 08:14:25.339644 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-8b196f49443f4479837a06e16e7aca6b on tablet 8b196f49443f4479837a06e16e7aca6b with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:25.403517 15369 raft_consensus.cc:1203] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.421777 17150 ts_tablet_manager.cc:902] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:25.432780 17150 tablet_copy_client.cc:308] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:25.440753 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 795dd1b66844485dad769acd4ec2b16f from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:57959)
I20240501 08:14:25.440935 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 795dd1b66844485dad769acd4ec2b16f from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:57959: session id = 3cd5ec97de8642dd8796fae63ea73e5a-795dd1b66844485dad769acd4ec2b16f
I20240501 08:14:25.445114 13090 tablet_copy_source_session.cc:205] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:25.447016 17150 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:25.451620 17150 tablet_copy_client.cc:783] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:25.451859 17150 tablet_copy_client.cc:647] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:25.453684 17150 tablet_copy_client.cc:523] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:25.456789 17150 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:25.490509 17150 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:25.490909 17150 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:25.491155 17150 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.034s user 0.030s sys 0.001s
I20240501 08:14:25.491866 17150 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.492158 17150 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:25.492408 17150 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:25.493404 17150 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240501 08:14:25.494102 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-795dd1b66844485dad769acd4ec2b16f received from {username='slave'} at 127.28.1.138:57959
I20240501 08:14:25.494271 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-795dd1b66844485dad769acd4ec2b16f on tablet 795dd1b66844485dad769acd4ec2b16f with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:25.576891 15103 raft_consensus.cc:1203] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.618057 16990 raft_consensus.cc:1050] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:25.619019 16990 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:25.621965 15103 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.622134 15236 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.622622 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.622911 12938 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.623057 16984 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.623456 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.626258 12803 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.626557 13332 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.627168 16964 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.627729 16984 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.635373 16984 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.636436 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.636509 12672 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.636154 15236 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.637255 17231 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.638557 17232 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.640260 15103 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.640316 12803 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.646250 12529 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.651659 16964 raft_consensus.cc:1050] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:25.652849 16964 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:25.657256 12938 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.660984 13335 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.662081 15236 raft_consensus.cc:1203] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.664203 13068 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.670516 17100 ts_tablet_manager.cc:902] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:25.676985 15236 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.677646 13335 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.678195 13070 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.678210 15369 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.678720 12936 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.679049 16990 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.679644 16990 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.680119 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.689996 16990 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.691555 12803 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.691849 12670 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.692829 17231 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.695458 17147 raft_consensus.cc:1050] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:25.696676 17147 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:25.702742 17231 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.712149 16990 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.723524 17232 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.725271 15369 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.730290 12938 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.730466 15103 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.730911 17013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.730999 12806 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.731750 17180 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.733891 17180 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.734485 12670 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.731349 12934 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.729228 12803 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.750824 12530 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.755589 12672 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.727620 17100 tablet_copy_client.cc:308] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:25.730515 15368 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.771268 17180 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.773231 12670 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.774144 13335 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.774648 17260 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.775218 13465 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.775174 17013 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.777065 17147 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.778697 15236 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.782636 15102 raft_consensus.cc:1203] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:25.790637 13332 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.796922 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet cd1c9882eb034801aa59324fe069461e from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:60059)
I20240501 08:14:25.797158 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet cd1c9882eb034801aa59324fe069461e from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:60059: session id = 35cb9fadf231408598c3877837a6149e-cd1c9882eb034801aa59324fe069461e
I20240501 08:14:25.798772 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.802735 12694 tablet_copy_source_session.cc:205] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:25.804258 17100 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:25.810307 13465 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:25.815936 15236 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.816807 13332 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.817368 16964 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.818419 16964 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.819379 15367 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.820572 16964 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.824149 12805 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.824177 12938 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.824787 17244 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.825414 17231 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.826597 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.827198 17244 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.829706 16990 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.834636 12938 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.836032 17169 raft_consensus.cc:1050] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:25.837352 17169 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:25.840988 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 16288057e4f24e8ebe4fd959d026d9d8 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:25.841063 12805 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.841701 13332 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.842468 12530 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.845331 15369 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.851384 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.858718 15236 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:25.866443 17100 tablet_copy_client.cc:783] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:25.866765 17100 tablet_copy_client.cc:647] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:25.869844 12651 tablet_service.cc:1514] Processing DeleteTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 with delete_type TABLET_DATA_TOMBSTONED (TS 56e1ccecd51040458c27911fa34a90a6 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39094
I20240501 08:14:25.875512 17147 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.881213 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.881510 12806 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.882333 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.887496 17271 tablet_replica.cc:330] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: stopping tablet replica
I20240501 08:14:25.888511 17271 raft_consensus.cc:2229] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:25.888960 17271 raft_consensus.cc:2258] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:25.890887 17271 ts_tablet_manager.cc:1875] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:25.892783 12531 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.909474 15369 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.917505 17271 ts_tablet_manager.cc:1888] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:25.918179 17271 log.cc:1196] T 16288057e4f24e8ebe4fd959d026d9d8 P 56e1ccecd51040458c27911fa34a90a6: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:25.919447 12516 catalog_manager.cc:4830] TS 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): tablet 16288057e4f24e8ebe4fd959d026d9d8 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:25.932168 15103 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.958462 15103 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.959264 17168 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.962896 12938 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.963739 17168 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.963994 15369 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.964582 17168 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240501 08:14:25.965946 13005 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 65466.2us
I20240501 08:14:25.968734 12803 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:25.969352 17168 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:25.972764 12670 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.979202 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:25.984484 12670 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.984552 15103 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.984547 15368 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.985059 17276 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.985824 17274 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.986315 17274 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:25.992938 12934 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:25.994230 13466 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.959992 17100 tablet_copy_client.cc:523] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:25.995609 17053 raft_consensus.cc:1050] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:25.996774 17053 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:25.998154 12936 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:25.999593 17100 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:26.000908 17147 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.003224 17274 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.004629 15236 raft_consensus.cc:1203] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:26.006439 17274 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.007824 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9f65863a31714bd19691d8eb5187ca91 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.008380 13466 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.008919 15103 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.012264 12934 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.014536 12670 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.015259 12529 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.020931 12786 tablet_service.cc:1514] Processing DeleteTablet for tablet 9f65863a31714bd19691d8eb5187ca91 with delete_type TABLET_DATA_TOMBSTONED (TS 1be03ffeb3384b42bfee6ffaf3f451c9 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:60684
I20240501 08:14:26.032917 13466 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.035212 12670 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.036602 15368 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.042183 17289 tablet_replica.cc:330] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: stopping tablet replica
I20240501 08:14:26.042776 17287 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.044548 17289 raft_consensus.cc:2229] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.046001 15368 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.046756 17289 raft_consensus.cc:2258] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.046686 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.053143 15103 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.053799 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.058765 13466 consensus_queue.cc:237] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:26.070444 15102 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.071264 13068 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.071276 16990 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.072554 15236 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.073340 12934 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.073370 16990 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.073953 17269 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.074962 13335 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.075474 16984 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.075789 17269 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.078186 12670 raft_consensus.cc:1261] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.078776 16984 consensus_queue.cc:1035] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.079317 16984 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.079777 15236 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.084928 12672 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.086117 12936 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.087159 13332 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.087779 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.091509 13335 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.097618 12938 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.097397 12531 catalog_manager.cc:5484] T 389d0cce047e48d7b702cb7245f60659 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.098984 13466 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.099567 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.102046 12786 tablet_service.cc:1514] Processing DeleteTablet for tablet 389d0cce047e48d7b702cb7245f60659 with delete_type TABLET_DATA_TOMBSTONED (TS 1be03ffeb3384b42bfee6ffaf3f451c9 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:60684
I20240501 08:14:26.103706 12803 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.104712 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.109701 17272 raft_consensus.cc:1050] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:26.109671 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 389d0cce047e48d7b702cb7245f60659 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.110821 17272 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:26.111284 15102 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.116601 13068 raft_consensus.cc:2932] T 389d0cce047e48d7b702cb7245f60659 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.126588 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.134660 17282 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.140995 17287 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.157644 12670 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.158242 15102 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.158350 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.159049 17275 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240501 08:14:26.160562 13269 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:43293, user_credentials={real_user=slave}} blocked reactor thread for 47102.4us
I20240501 08:14:26.163018 15368 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.163697 17275 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.165392 12938 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.165416 17058 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.166657 13465 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.168731 17272 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.168924 12936 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.169565 15103 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.172050 17289 ts_tablet_manager.cc:1875] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.169658 15368 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.182961 12806 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.183795 17275 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.184130 12936 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.187049 17275 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240501 08:14:26.188539 13270 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 219358us
I20240501 08:14:26.192654 13332 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.193286 12531 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.202541 12531 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.204219 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.203719 13466 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.216794 12672 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.218269 17272 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.219656 17269 raft_consensus.cc:1050] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:26.220937 17269 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:26.222611 15368 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.225423 12673 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.228605 13334 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.226552 15103 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.229305 13333 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.231338 17282 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.233950 17275 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.234768 17275 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.239152 16780 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.245275 15102 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.245898 12531 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:26.245908 13466 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.251211 13333 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.252427 13465 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.254753 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.255445 12803 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.267638 12938 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.268750 12673 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.271740 13202 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:26.275911 15368 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.276575 12934 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.277338 17053 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.277959 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.278051 17289 ts_tablet_manager.cc:1888] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.279217 15102 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.281220 13333 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.283246 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.283771 16661 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.285521 13200 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.286069 15236 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.286360 15367 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.286633 12934 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.287346 16964 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.288179 17238 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.288465 12673 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.288866 17238 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.289372 17232 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.289598 13333 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.290069 17289 log.cc:1196] T 9f65863a31714bd19691d8eb5187ca91 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:26.290329 16964 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.291251 17289 tablet_replica.cc:330] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: stopping tablet replica
I20240501 08:14:26.291462 17289 raft_consensus.cc:2229] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.291795 17289 raft_consensus.cc:2258] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.292172 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.293682 12517 catalog_manager.cc:4830] TS 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): tablet 9f65863a31714bd19691d8eb5187ca91 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:26.295763 16661 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.299741 12806 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.300480 16984 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.301012 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.304369 15366 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.311693 12527 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.314687 17289 ts_tablet_manager.cc:1875] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.315711 12671 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.319130 13465 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.321877 17316 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.324297 15102 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.324405 13466 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.318993 12786 tablet_service.cc:1514] Processing DeleteTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 9 (current committed config index is 10)) from {username='slave'} at 127.0.0.1:60684
I20240501 08:14:26.329784 15369 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.327325 12805 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.343750 12936 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.343942 13332 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.346161 16984 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.350719 12936 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.352798 17289 ts_tablet_manager.cc:1888] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.354410 17289 log.cc:1196] T 389d0cce047e48d7b702cb7245f60659 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/389d0cce047e48d7b702cb7245f60659
I20240501 08:14:26.355882 12517 catalog_manager.cc:4830] TS 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): tablet 389d0cce047e48d7b702cb7245f60659 (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:26.357270 17289 tablet_replica.cc:330] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: stopping tablet replica
I20240501 08:14:26.357712 17053 raft_consensus.cc:1050] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:26.357373 15236 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.358407 12529 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.359498 17289 raft_consensus.cc:2229] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.359316 17053 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:26.359982 17289 raft_consensus.cc:2258] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.361008 17289 ts_tablet_manager.cc:1875] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.367457 13200 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.368703 15366 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.372359 13335 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.381278 12670 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.381863 13068 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:26.383118 17325 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.383467 13201 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.385689 15366 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.385766 12805 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.386528 13465 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.386525 17260 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.386993 17013 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.388427 17013 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.388734 12671 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.393574 15103 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.394063 13331 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.393337 12803 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.395022 13332 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.396705 17260 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.397395 13465 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.401921 12672 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.402673 17272 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.404505 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e9b18b849c914fa5ba731892b7b88d6e with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.404003 15366 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.404412 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.405041 12671 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.414285 12531 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:26.415720 17316 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.020s
I20240501 08:14:26.417073 12936 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.418183 12673 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.418583 13465 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.419528 17058 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.420894 17058 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.421908 12918 tablet_service.cc:1514] Processing DeleteTablet for tablet e9b18b849c914fa5ba731892b7b88d6e with delete_type TABLET_DATA_TOMBSTONED (TS 5bf85222f4734b7885eb866efd27d5f2 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:44580
I20240501 08:14:26.422243 17272 raft_consensus.cc:1050] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:26.423374 17272 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:26.424607 17289 ts_tablet_manager.cc:1888] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.425390 17289 log.cc:1196] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:26.426314 12517 catalog_manager.cc:4830] TS 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:26.429121 15102 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.428966 13467 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:26.432099 15103 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.432088 15236 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.432206 15237 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.433221 12938 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.433151 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.433959 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.434091 16990 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.434757 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.439179 12803 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.439800 17269 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.449139 13333 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.450021 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.424276 12672 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.456732 12938 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.457464 17013 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.461313 17058 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.463325 16990 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.468082 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 85eb7c44993b4749b4b0a090d773bbef with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.468930 15237 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.471016 12531 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:26.476112 12651 tablet_service.cc:1514] Processing DeleteTablet for tablet 85eb7c44993b4749b4b0a090d773bbef with delete_type TABLET_DATA_TOMBSTONED (TS 56e1ccecd51040458c27911fa34a90a6 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39094
I20240501 08:14:26.475579 12938 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.478289 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.478606 12672 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.479198 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.479346 12805 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.480051 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.480806 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.482725 12803 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
W20240501 08:14:26.494164 13136 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.137:37989, user_credentials={real_user=slave}} blocked reactor thread for 98699.4us
I20240501 08:14:26.495836 17342 tablet_replica.cc:330] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: stopping tablet replica
I20240501 08:14:26.496534 17342 raft_consensus.cc:2229] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.497211 17342 raft_consensus.cc:2258] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.507589 15367 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.508487 17350 tablet_replica.cc:330] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: stopping tablet replica
I20240501 08:14:26.508790 17350 raft_consensus.cc:2229] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.509153 17350 raft_consensus.cc:2258] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.510195 17350 ts_tablet_manager.cc:1875] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.512517 16984 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.517733 17272 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240501 08:14:26.520684 13138 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 125078us
I20240501 08:14:26.522277 12802 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.522886 13331 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.523124 13334 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.524056 16984 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.524205 17272 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.524363 17325 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.525326 13200 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.525825 13334 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.530625 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.533668 16984 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.536489 15102 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.536849 13202 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.535480 17338 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.007s
I20240501 08:14:26.548415 15369 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:26.550320 17333 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.551198 17342 ts_tablet_manager.cc:1875] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.562922 17275 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.544040 15367 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.567440 17325 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.572727 13467 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.572832 12802 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.574985 12527 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:26.577652 12531 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:26.576380 13331 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.578523 13334 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.583209 13465 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.584803 12936 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.586791 12673 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.589236 12934 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.590143 12803 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.600560 15103 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.601642 15236 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.604949 13068 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:26.610428 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.610677 12802 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.611085 17260 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.611515 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.618337 15366 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.623289 13330 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.624228 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.629792 15236 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.630594 17260 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.633605 17272 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.637549 13330 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.638298 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 795dd1b66844485dad769acd4ec2b16f with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.639254 15237 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.644505 12530 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.645337 12672 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.645205 13465 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.645978 16780 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:26.647203 13201 consensus_queue.cc:237] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:26.647397 12671 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.650890 13330 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.651510 17337 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.652143 12671 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.655525 12918 tablet_service.cc:1514] Processing DeleteTablet for tablet 795dd1b66844485dad769acd4ec2b16f with delete_type TABLET_DATA_TOMBSTONED (TS 5bf85222f4734b7885eb866efd27d5f2 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:44580
I20240501 08:14:26.659402 12802 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.660377 15366 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.661126 16661 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.675411 13465 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.676517 17333 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.678838 17350 ts_tablet_manager.cc:1888] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.679005 17350 log.cc:1196] T 85eb7c44993b4749b4b0a090d773bbef P 56e1ccecd51040458c27911fa34a90a6: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:26.679951 12516 catalog_manager.cc:4830] TS 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): tablet 85eb7c44993b4749b4b0a090d773bbef (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:26.682327 12671 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.681746 17339 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.682933 16661 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.684590 12802 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.684700 13465 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.685834 17338 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.685429 12671 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.686067 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b196f49443f4479837a06e16e7aca6b with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.688051 15100 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.689234 15103 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.690879 12529 catalog_manager.cc:5484] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:26.693346 12802 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.694926 17275 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.695952 12918 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b196f49443f4479837a06e16e7aca6b with delete_type TABLET_DATA_TOMBSTONED (TS 5bf85222f4734b7885eb866efd27d5f2 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:44580
I20240501 08:14:26.696759 13331 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.697124 15102 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:26.700049 15367 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.710433 15100 raft_consensus.cc:1261] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.711942 17338 consensus_queue.cc:1035] T 8b196f49443f4479837a06e16e7aca6b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.713330 17342 ts_tablet_manager.cc:1888] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.713608 17342 log.cc:1196] T e9b18b849c914fa5ba731892b7b88d6e P 5bf85222f4734b7885eb866efd27d5f2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:26.713869 15100 raft_consensus.cc:2932] T 8b196f49443f4479837a06e16e7aca6b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:26.714637 17342 tablet_replica.cc:330] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: stopping tablet replica
I20240501 08:14:26.714787 12518 catalog_manager.cc:4830] TS 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): tablet e9b18b849c914fa5ba731892b7b88d6e (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:26.714911 17342 raft_consensus.cc:2229] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.715155 17342 raft_consensus.cc:2258] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.715914 17342 ts_tablet_manager.cc:1875] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.726953 17342 ts_tablet_manager.cc:1888] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.727190 17342 log.cc:1196] T 795dd1b66844485dad769acd4ec2b16f P 5bf85222f4734b7885eb866efd27d5f2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/795dd1b66844485dad769acd4ec2b16f
I20240501 08:14:26.728019 17342 tablet_replica.cc:330] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: stopping tablet replica
I20240501 08:14:26.728041 12518 catalog_manager.cc:4830] TS 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): tablet 795dd1b66844485dad769acd4ec2b16f (table rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]) successfully deleted
I20240501 08:14:26.728695 17342 raft_consensus.cc:2229] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.729382 17342 raft_consensus.cc:2258] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.730451 17342 ts_tablet_manager.cc:1875] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.738220 17342 ts_tablet_manager.cc:1888] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.738474 17342 log.cc:1196] T 8b196f49443f4479837a06e16e7aca6b P 5bf85222f4734b7885eb866efd27d5f2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/8b196f49443f4479837a06e16e7aca6b
I20240501 08:14:26.740180 12518 catalog_manager.cc:4830] TS 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): tablet 8b196f49443f4479837a06e16e7aca6b (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:26.757369 17100 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:26.757862 17100 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:26.758244 17100 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.759s user 0.048s sys 0.015s
I20240501 08:14:26.759066 17100 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:26.759322 17100 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:26.760236 17100 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:26.761319 17100 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:26.763944 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-cd1c9882eb034801aa59324fe069461e received from {username='slave'} at 127.28.1.136:60059
I20240501 08:14:26.766149 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 35cb9fadf231408598c3877837a6149e-cd1c9882eb034801aa59324fe069461e on tablet cd1c9882eb034801aa59324fe069461e with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:26.815202 13331 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:26.818048 15102 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.818179 15367 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.818282 12934 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.818847 12802 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.819331 17282 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.819864 17282 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.821062 17389 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.821602 17389 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:26.825445 13465 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.825448 15367 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.830713 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:26.832132 15102 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.839643 17280 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.008s
I20240501 08:14:26.839802 12529 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.843039 13201 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.845062 12651 tablet_service.cc:1514] Processing DeleteTablet for tablet b6828130a760491eb21fd3d346219748 with delete_type TABLET_DATA_TOMBSTONED (TS 56e1ccecd51040458c27911fa34a90a6 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39094
I20240501 08:14:26.845458 17350 tablet_replica.cc:330] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: stopping tablet replica
I20240501 08:14:26.845659 17350 raft_consensus.cc:2229] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:26.846174 17350 raft_consensus.cc:2258] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:26.847571 17350 ts_tablet_manager.cc:1875] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:26.850591 17302 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.010s
I20240501 08:14:26.839152 17282 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.857647 17350 ts_tablet_manager.cc:1888] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:26.857872 17350 log.cc:1196] T b6828130a760491eb21fd3d346219748 P 56e1ccecd51040458c27911fa34a90a6: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/b6828130a760491eb21fd3d346219748
I20240501 08:14:26.858736 12516 catalog_manager.cc:4830] TS 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): tablet b6828130a760491eb21fd3d346219748 (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:26.858407 13465 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.863255 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b6828130a760491eb21fd3d346219748 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:26.865080 12934 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:26.869328 12802 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.227377 15367 raft_consensus.cc:1203] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:27.718725 17441 raft_consensus.cc:1050] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:27.719846 17441 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:27.723403 13331 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.723624 13467 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.723701 15102 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.724198 12937 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.724210 17441 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.724332 15367 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.724946 12806 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.725262 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:27.725934 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.726584 17443 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.727262 17443 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.731480 17440 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.735858 17443 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.736274 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:27.752066 17441 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.753578 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.759060 12806 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.761119 12530 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.762286 13467 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.763428 15102 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.759815 15367 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.789891 13335 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.792847 12937 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.798434 12673 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:27.801276 15367 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.802181 13467 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.802106 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.802395 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.802624 13335 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.802987 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.803208 17446 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.803639 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.806280 17440 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.806882 12937 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.806932 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.807431 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.808403 15102 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:27.809082 17300 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:27.810787 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd1c9882eb034801aa59324fe069461e with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:27.809787 12937 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.811323 13465 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.812090 15367 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.814075 12524 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:27.816581 15102 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.817986 13331 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:27.825872 12786 tablet_service.cc:1514] Processing DeleteTablet for tablet cd1c9882eb034801aa59324fe069461e with delete_type TABLET_DATA_TOMBSTONED (TS 1be03ffeb3384b42bfee6ffaf3f451c9 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:60684
I20240501 08:14:27.826571 17465 tablet_replica.cc:330] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: stopping tablet replica
I20240501 08:14:27.827330 17465 raft_consensus.cc:2229] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:27.827639 17465 raft_consensus.cc:2258] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:27.828594 17465 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:27.834774 17465 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:27.834900 17465 log.cc:1196] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/cd1c9882eb034801aa59324fe069461e
I20240501 08:14:27.835578 12517 catalog_manager.cc:4830] TS 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): tablet cd1c9882eb034801aa59324fe069461e (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:28.182266 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:28.184509 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:28.186590 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:28.198436 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:28.203943 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:28.209955 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:28.217856 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:28.248852 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:28.256049 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:28.307107 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:28.420655 13465 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:28.424499 15237 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240501 08:14:28.424824 13400 consensus_peers.cc:485] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 16288057e4f24e8ebe4fd959d026d9d8. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:28.425052 15366 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.425113 16964 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.425993 17525 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.001s
I20240501 08:14:28.427337 12934 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.427505 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.428519 13335 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.429157 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.429837 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.431105 16984 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.433166 12806 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.434219 16964 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.437166 16984 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.440260 12806 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.442065 15366 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.445959 13335 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.453048 12531 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.454602 13070 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.455528 15237 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.462028 12934 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:28.486701 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:28.490088 15102 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240501 08:14:28.490250 13268 consensus_peers.cc:485] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cab182a0e126421c986101ea19df3e97. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:28.490383 12934 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:28.490933 17390 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.491590 12806 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:28.491654 17546 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.492228 17433 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.492319 12673 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:28.492856 13465 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:28.493616 17390 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.494143 17390 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.498543 13070 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:28.499173 17390 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:28.505722 17433 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.506872 15102 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.511186 12806 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.506412 13465 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.517547 12673 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.518862 12527 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.528573 12937 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.533995 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:28.562471 13070 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
W20240501 08:14:28.566108 13006 consensus_peers.cc:485] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9f65863a31714bd19691d8eb5187ca91. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:28.566856 15366 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.567065 15102 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.567608 12937 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.567936 17275 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.568336 17561 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.570312 17275 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.571314 12673 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.579353 17561 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.589385 17562 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.590020 15100 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.594403 13465 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.594961 17275 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.595172 13335 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:28.595799 17275 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:28.596156 12529 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.599159 12673 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.610524 15366 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.611196 12937 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.611299 13465 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.613075 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:28.893167 17601 ts_tablet_manager.cc:902] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:28.897145 17601 tablet_copy_client.cc:308] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:28.898057 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:45251)
I20240501 08:14:28.898229 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 16288057e4f24e8ebe4fd959d026d9d8 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:45251: session id = 8e309f48ce254660915fb9fc32649ab2-16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:28.900357 13488 tablet_copy_source_session.cc:205] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:28.901626 17601 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 16288057e4f24e8ebe4fd959d026d9d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:28.906718 17601 tablet_copy_client.cc:783] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:28.906989 17601 tablet_copy_client.cc:647] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:28.909029 17601 tablet_copy_client.cc:523] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:28.911947 17601 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:28.949059 17601 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:28.949528 17601 tablet_bootstrap.cc:492] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:28.949815 17601 ts_tablet_manager.cc:1367] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.038s user 0.035s sys 0.003s
I20240501 08:14:28.950619 17601 raft_consensus.cc:348] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:28.950922 17601 raft_consensus.cc:729] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:28.951155 17601 consensus_queue.cc:260] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:28.951712 17601 ts_tablet_manager.cc:1398] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:28.952523 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-16288057e4f24e8ebe4fd959d026d9d8 received from {username='slave'} at 127.28.1.137:45251
I20240501 08:14:28.952684 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-16288057e4f24e8ebe4fd959d026d9d8 on tablet 16288057e4f24e8ebe4fd959d026d9d8 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:29.068540 17607 ts_tablet_manager.cc:902] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:29.069756 17607 tablet_copy_client.cc:308] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:29.070453 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 9f65863a31714bd19691d8eb5187ca91 from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:57959)
I20240501 08:14:29.070631 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 9f65863a31714bd19691d8eb5187ca91 from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:57959: session id = 3cd5ec97de8642dd8796fae63ea73e5a-9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:29.072559 13090 tablet_copy_source_session.cc:205] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:29.073781 17607 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f65863a31714bd19691d8eb5187ca91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:29.078387 17607 tablet_copy_client.cc:783] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:29.078635 17607 tablet_copy_client.cc:647] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:29.080598 17607 tablet_copy_client.cc:523] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:29.081420 17612 ts_tablet_manager.cc:902] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:29.083086 17612 tablet_copy_client.cc:308] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:29.083757 17607 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:29.084133 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet cab182a0e126421c986101ea19df3e97 from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:46921)
I20240501 08:14:29.084302 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet cab182a0e126421c986101ea19df3e97 from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:46921: session id = 35cb9fadf231408598c3877837a6149e-cab182a0e126421c986101ea19df3e97
I20240501 08:14:29.086704 13355 tablet_copy_source_session.cc:205] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:29.088199 17612 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:29.094123 17612 tablet_copy_client.cc:783] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:29.094435 17612 tablet_copy_client.cc:647] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:29.096060 17612 tablet_copy_client.cc:523] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:29.098915 17612 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:29.165762 17612 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:29.166239 17612 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:29.166551 17612 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.068s user 0.049s sys 0.016s
I20240501 08:14:29.167374 17612 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:29.167696 17612 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:29.167941 17612 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:29.170607 17607 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:29.170985 17607 tablet_bootstrap.cc:492] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:29.171237 17607 ts_tablet_manager.cc:1367] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.088s user 0.048s sys 0.000s
I20240501 08:14:29.171870 17607 raft_consensus.cc:348] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:29.172225 17607 raft_consensus.cc:729] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:29.172497 17607 consensus_queue.cc:260] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:29.172641 17612 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.006s user 0.000s sys 0.006s
I20240501 08:14:29.173570 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-cab182a0e126421c986101ea19df3e97 received from {username='slave'} at 127.28.1.136:46921
I20240501 08:14:29.173635 17607 ts_tablet_manager.cc:1398] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240501 08:14:29.173730 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session 35cb9fadf231408598c3877837a6149e-cab182a0e126421c986101ea19df3e97 on tablet cab182a0e126421c986101ea19df3e97 with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:29.175212 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-9f65863a31714bd19691d8eb5187ca91 received from {username='slave'} at 127.28.1.138:57959
I20240501 08:14:29.175347 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-9f65863a31714bd19691d8eb5187ca91 on tablet 9f65863a31714bd19691d8eb5187ca91 with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:29.360790 15100 raft_consensus.cc:1203] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:29.482867 15237 raft_consensus.cc:1203] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:29.701643 15366 raft_consensus.cc:1203] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:29.742064 17535 raft_consensus.cc:1050] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:29.743338 17535 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:29.749163 15100 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.749614 15366 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.750418 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.750509 15237 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.751559 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.752139 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.760419 13335 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.760469 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.761113 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.761704 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.766587 12934 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.767263 17535 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.767701 12806 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.768436 17535 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.769354 17526 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.774214 12934 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.776029 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.783630 15237 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.789057 12529 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:29.800087 13335 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.805423 15367 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.806048 12802 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.810426 15100 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.822863 13465 consensus_queue.cc:237] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:29.825467 13335 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.825503 12802 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.826257 15100 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.826110 17680 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:29.826692 15367 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.826810 17679 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:29.827642 15237 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.827790 13068 raft_consensus.cc:1261] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:29.827085 17680 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.002s
I20240501 08:14:29.829067 17535 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:29.829313 17526 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:29.829952 17636 consensus_queue.cc:1035] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:29.832837 17535 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.836787 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 16288057e4f24e8ebe4fd959d026d9d8 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:29.836732 13068 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.837365 15100 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.837886 12802 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.841303 15367 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.841542 12528 catalog_manager.cc:5484] T 16288057e4f24e8ebe4fd959d026d9d8 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:29.842092 15237 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.843304 13335 raft_consensus.cc:2932] T 16288057e4f24e8ebe4fd959d026d9d8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:29.855715 12918 tablet_service.cc:1514] Processing DeleteTablet for tablet 16288057e4f24e8ebe4fd959d026d9d8 with delete_type TABLET_DATA_TOMBSTONED (TS 5bf85222f4734b7885eb866efd27d5f2 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:44580
I20240501 08:14:29.860947 17693 tablet_replica.cc:330] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: stopping tablet replica
I20240501 08:14:29.861186 17693 raft_consensus.cc:2229] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:29.861497 17693 raft_consensus.cc:2258] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:29.862423 17693 ts_tablet_manager.cc:1875] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:29.868412 17693 ts_tablet_manager.cc:1888] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:29.868548 17693 log.cc:1196] T 16288057e4f24e8ebe4fd959d026d9d8 P 5bf85222f4734b7885eb866efd27d5f2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/16288057e4f24e8ebe4fd959d026d9d8
I20240501 08:14:29.869351 12518 catalog_manager.cc:4830] TS 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): tablet 16288057e4f24e8ebe4fd959d026d9d8 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:29.976531 17562 raft_consensus.cc:1050] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:29.977726 17562 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:29.981482 15367 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.981535 15100 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.981935 15237 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.982547 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.982915 17570 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.983523 17570 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.992583 12934 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.992666 13465 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.993197 13335 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.993188 12673 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:29.993737 17715 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.994065 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.994395 17570 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:29.994933 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:30.008497 17715 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.010380 12673 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.011555 12934 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.014421 15367 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.016593 12525 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:30.016800 13465 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.018548 15237 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.023056 15100 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.027382 13068 consensus_queue.cc:237] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:30.009416 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.030647 15100 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.031423 15237 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.031380 17562 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.032158 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.032322 13331 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.032644 13465 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.033015 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.033941 15367 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.034065 17570 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.034565 17570 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.036074 12934 raft_consensus.cc:1261] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:30.036734 17563 consensus_queue.cc:1035] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:30.039304 17570 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.039804 13335 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.041316 13465 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.043386 15100 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.044860 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9f65863a31714bd19691d8eb5187ca91 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:30.045969 12527 catalog_manager.cc:5484] T 9f65863a31714bd19691d8eb5187ca91 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.050576 12651 tablet_service.cc:1514] Processing DeleteTablet for tablet 9f65863a31714bd19691d8eb5187ca91 with delete_type TABLET_DATA_TOMBSTONED (TS 56e1ccecd51040458c27911fa34a90a6 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:39094
I20240501 08:14:30.051278 12934 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.057484 15236 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.062304 17725 tablet_replica.cc:330] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: stopping tablet replica
I20240501 08:14:30.062052 15367 raft_consensus.cc:2932] T 9f65863a31714bd19691d8eb5187ca91 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:30.062733 17725 raft_consensus.cc:2229] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:30.063210 17725 raft_consensus.cc:2258] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:30.066469 17725 ts_tablet_manager.cc:1875] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:30.073179 17725 ts_tablet_manager.cc:1888] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:30.073304 17725 log.cc:1196] T 9f65863a31714bd19691d8eb5187ca91 P 56e1ccecd51040458c27911fa34a90a6: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-0/wal/wals/9f65863a31714bd19691d8eb5187ca91
I20240501 08:14:30.073999 12516 catalog_manager.cc:4830] TS 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349): tablet 9f65863a31714bd19691d8eb5187ca91 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:30.128764 17553 raft_consensus.cc:1050] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:30.129887 17553 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:30.133895 15367 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.136018 17668 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.137815 15100 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.138553 17669 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.140219 12934 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.140847 17668 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.142067 12802 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.143558 13068 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.145159 12673 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.146660 13465 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:30.147298 17742 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.147935 17668 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.148816 17668 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.149992 17744 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:30.152756 17669 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.153400 12802 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.158421 15367 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.159987 12527 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.167861 12934 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.170790 15100 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.178987 13465 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.182658 12671 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.184891 13068 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { replace: true } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.196347 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:30.199380 15367 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.199420 12934 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.199847 12671 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.200037 15100 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.199420 13465 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.200532 17669 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.200894 17749 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.201198 17553 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.201431 13068 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:30.201474 17743 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.201758 17752 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.201997 17751 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:30.207072 17669 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.208189 12934 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.208218 13465 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.209033 15100 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.211699 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cab182a0e126421c986101ea19df3e97 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:30.213511 15367 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.215109 12525 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:30.219812 12786 tablet_service.cc:1514] Processing DeleteTablet for tablet cab182a0e126421c986101ea19df3e97 with delete_type TABLET_DATA_TOMBSTONED (TS 1be03ffeb3384b42bfee6ffaf3f451c9 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:60684
I20240501 08:14:30.220748 17756 tablet_replica.cc:330] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: stopping tablet replica
I20240501 08:14:30.220963 17756 raft_consensus.cc:2229] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:30.221663 17756 raft_consensus.cc:2258] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:30.223300 12671 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.224694 17756 ts_tablet_manager.cc:1875] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:30.229516 17756 ts_tablet_manager.cc:1888] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:30.229645 17756 log.cc:1196] T cab182a0e126421c986101ea19df3e97 P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-1/wal/wals/cab182a0e126421c986101ea19df3e97
I20240501 08:14:30.230342 12517 catalog_manager.cc:4830] TS 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): tablet cab182a0e126421c986101ea19df3e97 (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:30.230716 13068 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:30.543752 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:30.550058 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:30.564831 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:30.603344 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:30.652118 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:30.653034 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:30.662451 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:30.667194 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:30.681778 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:30.699251 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:30.978030 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:31.012626 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:31.013986 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:31.025286 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:31.041743 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:31.053803 13447 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:31.070128 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:31.081348 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:31.135318 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:31.145792 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:31.285351 12671 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } }
I20240501 08:14:31.293164 13201 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:31.302775 13331 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.304520 17652 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.001s
W20240501 08:14:31.305043 12608 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130:45135): Couldn't send request to peer 1be03ffeb3384b42bfee6ffaf3f451c9. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.306245 15366 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.307302 17736 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.309434 15102 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.310050 17736 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.313880 13335 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.314599 17736 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.323761 13068 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:31.325814 12934 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.325831 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.327250 13465 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.327606 17861 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.328519 17861 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.332885 17736 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.346985 12934 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.347734 17652 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.347254 17861 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.350126 13465 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.350853 17652 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.352243 13467 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.353135 12934 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.354095 15102 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
W20240501 08:14:31.358898 13137 consensus_peers.cc:485] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad -> Peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Couldn't send request to peer f59d8b4fd79049b6b40dcecd13d0be9d. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.366950 12522 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.369828 12671 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.370561 17652 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.370700 13335 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.371295 13070 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
W20240501 08:14:31.399387 13006 consensus_peers.cc:485] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e9b18b849c914fa5ba731892b7b88d6e. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.401043 17652 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.404762 13330 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.410738 15102 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.410718 12937 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.410951 13467 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.414546 15367 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.415285 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.419057 12527 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:31.426940 12802 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.427702 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.429440 12671 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.439427 12673 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.440193 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.440757 13467 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.441884 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.448768 13331 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.449389 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.459285 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240501 08:14:31.468858 13136 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.137:37989, user_credentials={real_user=slave}} blocked reactor thread for 168458us
I20240501 08:14:31.470500 15369 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.475248 17906 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.475986 15102 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.486390 17719 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.487993 17906 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.491698 12806 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.494705 15368 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.508939 15102 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.535169 15100 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
W20240501 08:14:31.554929 13005 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.130:45135, user_credentials={real_user=slave}} blocked reactor thread for 52283.9us
I20240501 08:14:31.561053 12530 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.564134 15368 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.579125 12671 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.582617 13467 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.586786 15366 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } }
I20240501 08:14:31.615456 13335 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:31.646629 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:31.649412 15366 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240501 08:14:31.650118 13270 consensus_peers.cc:485] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.650557 17935 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.650856 12934 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.651369 12671 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.651544 17935 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.651934 15100 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.652897 13068 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.653308 13467 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.653486 17749 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.655359 17749 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.655668 17752 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.656299 17752 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.657030 12937 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:31.682345 17935 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.685902 13201 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.685714 13467 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.686345 13465 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.686492 12671 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.686900 17682 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.687367 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.690292 12802 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.691310 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.691918 15366 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.693235 12524 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.696205 13070 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.698745 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:31.700565 12871 consensus_peers.cc:485] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1226795e4f534366874b23270d086334. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.701913 12671 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.709757 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:31.717343 13268 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.137:37989, user_credentials={real_user=slave}} blocked reactor thread for 61917us
I20240501 08:14:31.722165 13335 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.724012 15366 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.726645 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.735685 17682 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.736148 15100 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.741103 13335 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.741276 12673 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.746279 12530 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:31.747278 12806 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
W20240501 08:14:31.752940 13271 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.135:43031, user_credentials={real_user=slave}} blocked reactor thread for 61961.6us
I20240501 08:14:31.754089 12937 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:31.755712 15366 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.757099 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.755620 13465 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:31.768572 13467 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:31.776767 15366 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.776871 15237 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.777490 17535 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240501 08:14:31.778062 13401 consensus_peers.cc:485] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 -> Peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Couldn't send request to peer f59d8b4fd79049b6b40dcecd13d0be9d. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.778062 17535 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.797365 13201 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.798691 17526 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.800056 13335 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.800635 17535 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.801710 17526 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.802488 13201 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.809350 12524 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.814908 12806 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.815124 12937 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.815673 17535 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.816331 17535 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.818559 12806 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.826534 15237 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.834702 17979 ts_tablet_manager.cc:902] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:31.835927 17978 ts_tablet_manager.cc:902] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Initiating tablet copy from peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529)
I20240501 08:14:31.840210 15366 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.853221 12937 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.863184 13335 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:31.869941 17979 tablet_copy_client.cc:308] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:31.870985 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet cd1c9882eb034801aa59324fe069461e from peer 1be03ffeb3384b42bfee6ffaf3f451c9 ({username='slave'} at 127.28.1.130:51563)
I20240501 08:14:31.871181 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet cd1c9882eb034801aa59324fe069461e from peer 1be03ffeb3384b42bfee6ffaf3f451c9 at {username='slave'} at 127.28.1.130:51563: session id = 1be03ffeb3384b42bfee6ffaf3f451c9-cd1c9882eb034801aa59324fe069461e
I20240501 08:14:31.875080 12694 tablet_copy_source_session.cc:205] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:31.884306 17979 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:31.896410 17979 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240501 08:14:31.896879 17979 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:31.901049 17979 tablet_copy_client.cc:783] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:31.901392 17979 tablet_copy_client.cc:647] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:31.902295 17978 tablet_copy_client.cc:308] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.133:42529
I20240501 08:14:31.903409 13068 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:31.904023 17979 tablet_copy_client.cc:523] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:31.904165 13224 tablet_copy_service.cc:140] P fa5b68f497b84b31ac35e5549b2af1ad: Received BeginTabletCopySession request for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer f59d8b4fd79049b6b40dcecd13d0be9d ({username='slave'} at 127.28.1.132:60733)
I20240501 08:14:31.904328 13224 tablet_copy_service.cc:161] P fa5b68f497b84b31ac35e5549b2af1ad: Beginning new tablet copy session on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b from peer f59d8b4fd79049b6b40dcecd13d0be9d at {username='slave'} at 127.28.1.132:60733: session id = f59d8b4fd79049b6b40dcecd13d0be9d-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:31.906536 13224 tablet_copy_source_session.cc:205] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:31.907052 15237 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.907359 15100 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.908149 17927 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.908679 17979 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap starting.
I20240501 08:14:31.908782 17927 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240501 08:14:31.912256 13003 consensus_peers.cc:485] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): Couldn't send request to peer 35cb9fadf231408598c3877837a6149e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 795dd1b66844485dad769acd4ec2b16f. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.913481 17978 ts_tablet_manager.cc:1875] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:31.921384 13335 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.922880 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.923796 12806 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.924481 12671 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:31.925690 17927 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.926244 17927 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.927250 17978 ts_tablet_manager.cc:1888] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:31.928092 17978 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:31.928462 17570 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.929078 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:31.934520 17978 tablet_copy_client.cc:783] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:31.934860 17978 tablet_copy_client.cc:647] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:31.942639 17978 tablet_copy_client.cc:523] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:31.943169 17570 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:31.947440 12671 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:31.948355 13335 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:31.954689 12524 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:31.954860 12937 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:31.947153 17978 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:31.961695 12806 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
W20240501 08:14:31.973991 12873 consensus_peers.cc:485] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:31.975128 15235 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.975841 17682 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.980226 13331 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.981158 17962 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:31.984216 15102 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:31.993744 13465 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:31.994506 17682 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.005324 13068 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.005373 15237 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:32.005373 13467 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } } }
I20240501 08:14:32.006081 17682 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.020782 12671 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.022042 17682 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.031819 12806 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.033527 17962 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.038455 12673 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:32.046519 17962 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.056033 12671 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.062281 12524 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.062808 17979 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.063303 17979 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Bootstrap complete.
I20240501 08:14:32.066730 17979 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent bootstrapping tablet: real 0.158s user 0.057s sys 0.003s
I20240501 08:14:32.067586 12806 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.068888 17979 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } }
I20240501 08:14:32.069238 17979 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1be03ffeb3384b42bfee6ffaf3f451c9, State: Initialized, Role: LEARNER
I20240501 08:14:32.069602 17979 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } }
I20240501 08:14:32.070423 17979 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240501 08:14:32.072201 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 1be03ffeb3384b42bfee6ffaf3f451c9-cd1c9882eb034801aa59324fe069461e received from {username='slave'} at 127.28.1.130:51563
I20240501 08:14:32.072367 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 1be03ffeb3384b42bfee6ffaf3f451c9-cd1c9882eb034801aa59324fe069461e on tablet cd1c9882eb034801aa59324fe069461e with peer 1be03ffeb3384b42bfee6ffaf3f451c9
I20240501 08:14:32.079171 18020 ts_tablet_manager.cc:902] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:32.084040 15237 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.087460 17978 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.088527 17978 tablet_bootstrap.cc:492] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap complete.
I20240501 08:14:32.088927 17978 ts_tablet_manager.cc:1367] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.142s user 0.055s sys 0.008s
I20240501 08:14:32.089840 17978 raft_consensus.cc:348] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:32.090252 17978 raft_consensus.cc:729] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: LEARNER
I20240501 08:14:32.090561 17978 consensus_queue.cc:260] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:32.093783 13467 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.094636 15235 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.095430 18004 ts_tablet_manager.cc:902] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:32.096117 17861 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.096769 13070 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.097496 17861 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:32.097776 12606 consensus_peers.cc:485] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2ede6a5ca6654aceb2a24e1592fc1bc8. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.101660 13467 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.106202 17978 ts_tablet_manager.cc:1398] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.017s user 0.002s sys 0.001s
I20240501 08:14:32.107510 13224 tablet_copy_service.cc:342] P fa5b68f497b84b31ac35e5549b2af1ad: Request end of tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b received from {username='slave'} at 127.28.1.132:60733
I20240501 08:14:32.107651 13224 tablet_copy_service.cc:433] P fa5b68f497b84b31ac35e5549b2af1ad: ending tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b on tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with peer f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:32.136090 13331 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.144234 18031 ts_tablet_manager.cc:902] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:32.161428 13201 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.162439 17861 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.164036 12937 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.164119 13331 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.164199 12806 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.164806 17861 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.165407 17861 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.168397 18023 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.182739 17890 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.184798 12937 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.185215 12806 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.188810 13335 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.194293 12531 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.214308 15237 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.214550 13465 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.215440 13201 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } }
I20240501 08:14:32.221290 17978 ts_tablet_manager.cc:902] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Initiating tablet copy from peer b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031)
I20240501 08:14:32.230903 18031 tablet_copy_client.cc:308] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:32.235270 18004 tablet_copy_client.cc:308] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:32.236202 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet e9b18b849c914fa5ba731892b7b88d6e from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:57959)
I20240501 08:14:32.236379 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet e9b18b849c914fa5ba731892b7b88d6e from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:57959: session id = 3cd5ec97de8642dd8796fae63ea73e5a-e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:32.239089 13090 tablet_copy_source_session.cc:205] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.241140 12934 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
W20240501 08:14:32.245620 12871 consensus_peers.cc:485] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): Couldn't send request to peer 8e309f48ce254660915fb9fc32649ab2. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1226795e4f534366874b23270d086334. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.246907 15366 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.247252 18004 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b18b849c914fa5ba731892b7b88d6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.247862 13331 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.248474 13202 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.248889 18019 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.249467 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.249903 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 1226795e4f534366874b23270d086334 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:40137)
I20240501 08:14:32.250087 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 1226795e4f534366874b23270d086334 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:40137: session id = 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334
I20240501 08:14:32.251741 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.253764 12959 tablet_copy_source_session.cc:205] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.254014 13465 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.254598 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.255688 18031 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
W20240501 08:14:32.255776 12872 consensus_peers.cc:485] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 -> Peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Couldn't send request to peer f59d8b4fd79049b6b40dcecd13d0be9d. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.256645 18022 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.261701 15366 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.263036 18031 tablet_copy_client.cc:783] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.263325 18031 tablet_copy_client.cc:647] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.264899 18004 tablet_copy_client.cc:783] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.265151 18004 tablet_copy_client.cc:647] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.265465 12673 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.266539 12806 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:32.266832 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.267402 18031 tablet_copy_client.cc:523] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.268388 18019 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:32.268666 12673 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.271282 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:32.270821 13331 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.272612 13202 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.275457 13465 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.276783 13068 raft_consensus.cc:1203] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:32.277716 18004 tablet_copy_client.cc:523] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.279368 12531 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.283020 12806 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:32.284737 18004 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:32.303738 18064 ts_tablet_manager.cc:902] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:32.315165 12806 raft_consensus.cc:1203] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:32.320493 18020 tablet_copy_client.cc:308] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:32.324819 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet cab182a0e126421c986101ea19df3e97 from peer fa5b68f497b84b31ac35e5549b2af1ad ({username='slave'} at 127.28.1.133:47379)
I20240501 08:14:32.325042 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet cab182a0e126421c986101ea19df3e97 from peer fa5b68f497b84b31ac35e5549b2af1ad at {username='slave'} at 127.28.1.133:47379: session id = fa5b68f497b84b31ac35e5549b2af1ad-cab182a0e126421c986101ea19df3e97
I20240501 08:14:32.325953 17978 tablet_copy_client.cc:308] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.135:43031
I20240501 08:14:32.327926 13355 tablet_copy_source_session.cc:205] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.328639 13488 tablet_copy_service.cc:140] P b17fe4a1f54c4655b85eb663fcff1b93: Received BeginTabletCopySession request for tablet 85eb7c44993b4749b4b0a090d773bbef from peer f59d8b4fd79049b6b40dcecd13d0be9d ({username='slave'} at 127.28.1.132:48179)
I20240501 08:14:32.328830 13488 tablet_copy_service.cc:161] P b17fe4a1f54c4655b85eb663fcff1b93: Beginning new tablet copy session on tablet 85eb7c44993b4749b4b0a090d773bbef from peer f59d8b4fd79049b6b40dcecd13d0be9d at {username='slave'} at 127.28.1.132:48179: session id = f59d8b4fd79049b6b40dcecd13d0be9d-85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:32.331626 13488 tablet_copy_source_session.cc:205] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.333940 18020 ts_tablet_manager.cc:1875] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:32.334347 17978 ts_tablet_manager.cc:1875] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:32.341979 17978 ts_tablet_manager.cc:1888] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:32.342466 17978 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85eb7c44993b4749b4b0a090d773bbef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.350107 17978 tablet_copy_client.cc:783] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.354382 17978 tablet_copy_client.cc:647] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.356752 13068 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:32.357910 18020 ts_tablet_manager.cc:1888] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:32.358234 18020 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab182a0e126421c986101ea19df3e97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.369810 12673 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.369956 15366 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.370368 18020 tablet_copy_client.cc:783] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.370677 18020 tablet_copy_client.cc:647] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.370944 12806 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.371026 17570 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.372208 17927 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.372750 17927 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240501 08:14:32.376338 13006 consensus_peers.cc:485] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e8782ed1d2a5492e99b9bf6bd4b1597a. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.376988 12937 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.378576 17570 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.385207 18020 tablet_copy_client.cc:523] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.394209 12671 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:32.396962 18020 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
W20240501 08:14:32.405377 12608 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.406471 12806 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.407022 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.410256 17570 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.414491 12934 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.415302 15100 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.416078 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.421239 17890 raft_consensus.cc:1013] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: attempt to promote peer 1be03ffeb3384b42bfee6ffaf3f451c9: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240501 08:14:32.420939 12531 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.426582 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.426759 13465 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.427553 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.427896 13331 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.428102 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.428537 18023 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.432142 12934 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.432827 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.434038 15366 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.435201 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.450321 18049 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.457048 13467 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.464047 12802 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.468014 12934 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.468621 13331 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.480248 12673 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.482995 17978 tablet_copy_client.cc:523] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.484005 12531 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.487888 13335 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.488801 13466 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:32.489333 17719 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.489637 17927 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:32.506973 15100 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.512708 17890 raft_consensus.cc:1050] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 1be03ffeb3384b42bfee6ffaf3f451c9 to VOTER
I20240501 08:14:32.513958 17890 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
W20240501 08:14:32.517774 12608 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:32.518592 12806 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.519577 18087 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:32.520236 12806 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.520416 15367 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.521337 15368 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.521905 18087 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.523123 15368 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.534055 13331 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.535028 18087 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:32.535490 12937 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.536244 13467 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.536206 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:32.537065 18087 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:32.546440 13330 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.546296 18049 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.553253 12531 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.564311 13068 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.568277 18023 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.572707 18004 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.571689 13467 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.573247 18004 tablet_bootstrap.cc:492] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:32.574965 18004 ts_tablet_manager.cc:1367] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.290s user 0.054s sys 0.007s
I20240501 08:14:32.575783 18004 raft_consensus.cc:348] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:32.577577 18004 raft_consensus.cc:729] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:32.577819 18004 consensus_queue.cc:260] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:32.580771 18004 ts_tablet_manager.cc:1398] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240501 08:14:32.581552 12806 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.582926 15366 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.587646 18064 tablet_copy_client.cc:308] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:32.588677 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet 795dd1b66844485dad769acd4ec2b16f from peer 35cb9fadf231408598c3877837a6149e ({username='slave'} at 127.28.1.136:54165)
I20240501 08:14:32.588872 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet 795dd1b66844485dad769acd4ec2b16f from peer 35cb9fadf231408598c3877837a6149e at {username='slave'} at 127.28.1.136:54165: session id = 35cb9fadf231408598c3877837a6149e-795dd1b66844485dad769acd4ec2b16f
I20240501 08:14:32.589300 13466 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.589690 12937 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.591718 13090 tablet_copy_source_session.cc:205] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.592659 17978 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:32.586337 13335 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:32.597148 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-e9b18b849c914fa5ba731892b7b88d6e received from {username='slave'} at 127.28.1.138:57959
I20240501 08:14:32.597282 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-e9b18b849c914fa5ba731892b7b88d6e on tablet e9b18b849c914fa5ba731892b7b88d6e with peer 3cd5ec97de8642dd8796fae63ea73e5a
I20240501 08:14:32.597446 18064 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 795dd1b66844485dad769acd4ec2b16f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.607301 15102 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:32.608842 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:32.611215 18099 ts_tablet_manager.cc:902] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:32.579746 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: NON_VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: true } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.612360 15101 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.615989 18096 ts_tablet_manager.cc:902] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:32.643710 18064 tablet_copy_client.cc:783] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.645959 13068 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 1be03ffeb3384b42bfee6ffaf3f451c9 (127.28.1.130) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:32.662671 18064 tablet_copy_client.cc:647] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.664840 18064 tablet_copy_client.cc:523] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.674923 18096 tablet_copy_client.cc:308] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:32.675108 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240501 08:14:32.675685 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:32.676043 18031 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.405s user 0.053s sys 0.004s
I20240501 08:14:32.676061 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 2b098b5bce924e6ea5897e463451b964 from peer fa5b68f497b84b31ac35e5549b2af1ad ({username='slave'} at 127.28.1.133:48547)
I20240501 08:14:32.676214 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 2b098b5bce924e6ea5897e463451b964 from peer fa5b68f497b84b31ac35e5549b2af1ad at {username='slave'} at 127.28.1.133:48547: session id = fa5b68f497b84b31ac35e5549b2af1ad-2b098b5bce924e6ea5897e463451b964
I20240501 08:14:32.676946 18113 ts_tablet_manager.cc:902] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:32.680614 12959 tablet_copy_source_session.cc:205] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.682339 18096 ts_tablet_manager.cc:1875] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:32.682830 18031 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:32.683482 18031 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:32.683739 18031 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:32.688166 18099 tablet_copy_client.cc:308] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:32.689311 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:52199)
I20240501 08:14:32.689373 18064 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Bootstrap starting.
I20240501 08:14:32.689499 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:52199: session id = 8e309f48ce254660915fb9fc32649ab2-2ede6a5ca6654aceb2a24e1592fc1bc8
I20240501 08:14:32.691534 12694 tablet_copy_source_session.cc:205] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.696462 18099 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.697127 18096 ts_tablet_manager.cc:1888] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:32.697400 18096 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b098b5bce924e6ea5897e463451b964. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.698170 18031 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.022s user 0.002s sys 0.000s
I20240501 08:14:32.704097 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334 received from {username='slave'} at 127.28.1.137:40137
I20240501 08:14:32.704260 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334 on tablet 1226795e4f534366874b23270d086334 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:32.708549 18031 tablet_replica.cc:330] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: stopping tablet replica
I20240501 08:14:32.710673 18031 raft_consensus.cc:2229] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Raft consensus shutting down.
I20240501 08:14:32.711038 18031 pending_rounds.cc:62] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Trying to abort 1 pending ops.
I20240501 08:14:32.711138 18099 tablet_copy_client.cc:783] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.711416 18099 tablet_copy_client.cc:647] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.711313 18031 pending_rounds.cc:68] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Aborting op as it isn't in flight: id { term: 1 index: 9 } timestamp: 7022802011096760320 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "1226795e4f534366874b23270d086334" old_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } } new_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } } }
I20240501 08:14:32.712193 18031 raft_consensus.cc:2894] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Aborting config change with OpId 1.9: Aborted: Op aborted
I20240501 08:14:32.712385 18031 raft_consensus.cc:2866] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240501 08:14:32.712525 18031 raft_consensus.cc:2258] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Raft consensus is shut down!
I20240501 08:14:32.714586 18031 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:32.720436 18099 tablet_copy_client.cc:523] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.728966 18096 tablet_copy_client.cc:783] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.729116 18099 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:32.729305 18096 tablet_copy_client.cc:647] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.731369 18096 tablet_copy_client.cc:523] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.734982 18096 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:32.736022 18031 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:32.736279 18031 log.cc:1196] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/wals/1226795e4f534366874b23270d086334
I20240501 08:14:32.737190 18031 ts_tablet_manager.cc:902] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Initiating tablet copy from peer 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689)
I20240501 08:14:32.751910 18113 tablet_copy_client.cc:308] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:32.752745 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 1226795e4f534366874b23270d086334 from peer f59d8b4fd79049b6b40dcecd13d0be9d ({username='slave'} at 127.28.1.132:35353)
I20240501 08:14:32.752990 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 1226795e4f534366874b23270d086334 from peer f59d8b4fd79049b6b40dcecd13d0be9d at {username='slave'} at 127.28.1.132:35353: session id = f59d8b4fd79049b6b40dcecd13d0be9d-1226795e4f534366874b23270d086334
I20240501 08:14:32.758249 12959 tablet_copy_source_session.cc:205] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.759794 18113 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:32.772258 17906 raft_consensus.cc:1050] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad: attempting to promote NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d to VOTER
I20240501 08:14:32.773291 18113 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:32.773617 18113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.773578 17906 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:32.780969 18113 tablet_copy_client.cc:783] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.781289 18113 tablet_copy_client.cc:647] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.797293 18113 tablet_copy_client.cc:523] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.802436 15366 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.803061 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.804937 18113 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:32.812363 15101 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.813230 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.824270 13335 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.825119 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.831202 17978 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.831669 17978 tablet_bootstrap.cc:492] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap complete.
I20240501 08:14:32.831983 17978 ts_tablet_manager.cc:1367] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.239s user 0.049s sys 0.004s
I20240501 08:14:32.832834 17978 raft_consensus.cc:348] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:32.833161 17978 raft_consensus.cc:729] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: LEARNER
I20240501 08:14:32.833381 17978 consensus_queue.cc:260] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:32.835318 12937 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.835402 13068 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.837386 18031 tablet_copy_client.cc:308] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.131:38689
I20240501 08:14:32.838116 12959 tablet_copy_service.cc:140] P 5bf85222f4734b7885eb866efd27d5f2: Received BeginTabletCopySession request for tablet 1226795e4f534366874b23270d086334 from peer 8e309f48ce254660915fb9fc32649ab2 ({username='slave'} at 127.28.1.137:40137)
I20240501 08:14:32.838279 12959 tablet_copy_service.cc:161] P 5bf85222f4734b7885eb866efd27d5f2: Beginning new tablet copy session on tablet 1226795e4f534366874b23270d086334 from peer 8e309f48ce254660915fb9fc32649ab2 at {username='slave'} at 127.28.1.137:40137: session id = 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334
I20240501 08:14:32.841815 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.842679 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.844611 18004 ts_tablet_manager.cc:902] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Initiating tablet copy from peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293)
I20240501 08:14:32.849445 17978 ts_tablet_manager.cc:1398] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.017s user 0.002s sys 0.000s
I20240501 08:14:32.851881 12959 tablet_copy_source_session.cc:205] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:32.854683 13488 tablet_copy_service.cc:342] P b17fe4a1f54c4655b85eb663fcff1b93: Request end of tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-85eb7c44993b4749b4b0a090d773bbef received from {username='slave'} at 127.28.1.132:48179
I20240501 08:14:32.854825 13488 tablet_copy_service.cc:433] P b17fe4a1f54c4655b85eb663fcff1b93: ending tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-85eb7c44993b4749b4b0a090d773bbef on tablet 85eb7c44993b4749b4b0a090d773bbef with peer f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:32.859855 15237 raft_consensus.cc:1203] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:32.870442 18031 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Deleting tablet data with delete state TABLET_DATA_COPYING
W20240501 08:14:32.885727 13138 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 49466.5us
W20240501 08:14:32.887468 13137 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.132:43293, user_credentials={real_user=slave}} blocked reactor thread for 51557.1us
I20240501 08:14:32.888422 12673 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.894091 18031 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240501 08:14:32.894495 18031 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1226795e4f534366874b23270d086334. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:32.901938 13466 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:32.902855 18099 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.903299 18099 tablet_bootstrap.cc:492] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:32.903625 18099 ts_tablet_manager.cc:1367] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.180s user 0.073s sys 0.023s
I20240501 08:14:32.908547 18031 tablet_copy_client.cc:783] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:32.908795 18031 tablet_copy_client.cc:647] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:32.909335 18099 raft_consensus.cc:348] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:32.909662 18099 raft_consensus.cc:729] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:32.912447 18031 tablet_copy_client.cc:523] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:32.920784 17906 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.921873 18151 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:32.924708 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap starting.
I20240501 08:14:32.911684 18099 consensus_queue.cc:260] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } }
I20240501 08:14:32.930164 18099 ts_tablet_manager.cc:1398] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.026s user 0.005s sys 0.000s
I20240501 08:14:32.933104 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-2ede6a5ca6654aceb2a24e1592fc1bc8 received from {username='slave'} at 127.28.1.137:52199
I20240501 08:14:32.933255 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-2ede6a5ca6654aceb2a24e1592fc1bc8 on tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:32.934080 18156 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.951695 15368 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.955940 15101 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.963418 12531 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.971779 12937 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.979879 13335 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:32.985965 18064 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:32.989173 15101 raft_consensus.cc:1203] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:32.996968 18064 tablet_bootstrap.cc:492] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Bootstrap complete.
I20240501 08:14:32.997334 18064 ts_tablet_manager.cc:1367] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Time spent bootstrapping tablet: real 0.308s user 0.055s sys 0.002s
I20240501 08:14:32.998147 18064 raft_consensus.cc:348] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:32.998473 18064 raft_consensus.cc:729] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 35cb9fadf231408598c3877837a6149e, State: Initialized, Role: LEARNER
I20240501 08:14:32.998715 18064 consensus_queue.cc:260] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: NON_VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: true } }
I20240501 08:14:33.002269 18113 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=9 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.002741 18113 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap complete.
I20240501 08:14:33.003062 18113 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.198s user 0.046s sys 0.006s
I20240501 08:14:33.003763 13466 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.011852 13070 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.012677 18064 ts_tablet_manager.cc:1398] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e: Time spent starting tablet: real 0.015s user 0.002s sys 0.000s
I20240501 08:14:33.013724 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=9 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.014330 18031 tablet_bootstrap.cc:492] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Bootstrap complete.
I20240501 08:14:33.014955 18031 ts_tablet_manager.cc:1367] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Time spent bootstrapping tablet: real 0.090s user 0.054s sys 0.005s
I20240501 08:14:33.016059 18031 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:33.016398 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 35cb9fadf231408598c3877837a6149e-795dd1b66844485dad769acd4ec2b16f received from {username='slave'} at 127.28.1.136:54165
I20240501 08:14:33.016495 18031 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8e309f48ce254660915fb9fc32649ab2, State: Initialized, Role: LEARNER
I20240501 08:14:33.016522 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 35cb9fadf231408598c3877837a6149e-795dd1b66844485dad769acd4ec2b16f on tablet 795dd1b66844485dad769acd4ec2b16f with peer 35cb9fadf231408598c3877837a6149e
I20240501 08:14:33.016722 18031 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:33.020509 18031 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240501 08:14:33.019428 18113 raft_consensus.cc:348] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:33.021003 18113 raft_consensus.cc:729] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: LEARNER
I20240501 08:14:33.021234 18113 consensus_queue.cc:260] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: NON_VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:33.024209 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334 received from {username='slave'} at 127.28.1.137:40137
I20240501 08:14:33.024354 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session 8e309f48ce254660915fb9fc32649ab2-1226795e4f534366874b23270d086334 on tablet 1226795e4f534366874b23270d086334 with peer 8e309f48ce254660915fb9fc32649ab2
I20240501 08:14:33.025008 18113 ts_tablet_manager.cc:1398] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.022s user 0.005s sys 0.000s
I20240501 08:14:33.028028 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-1226795e4f534366874b23270d086334 received from {username='slave'} at 127.28.1.132:35353
I20240501 08:14:33.028187 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-1226795e4f534366874b23270d086334 on tablet 1226795e4f534366874b23270d086334 with peer f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:33.023591 12673 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.040408 13201 consensus_queue.cc:237] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:33.047413 12937 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.049636 15101 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.051741 17906 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.052856 17883 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.060072 15366 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.060931 17906 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.064476 13335 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.066116 18156 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.069944 18004 tablet_copy_client.cc:308] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.132:43293
I20240501 08:14:33.071110 13090 tablet_copy_service.cc:140] P f59d8b4fd79049b6b40dcecd13d0be9d: Received BeginTabletCopySession request for tablet e8782ed1d2a5492e99b9bf6bd4b1597a from peer 3cd5ec97de8642dd8796fae63ea73e5a ({username='slave'} at 127.28.1.138:57959)
I20240501 08:14:33.071285 13090 tablet_copy_service.cc:161] P f59d8b4fd79049b6b40dcecd13d0be9d: Beginning new tablet copy session on tablet e8782ed1d2a5492e99b9bf6bd4b1597a from peer 3cd5ec97de8642dd8796fae63ea73e5a at {username='slave'} at 127.28.1.138:57959: session id = 3cd5ec97de8642dd8796fae63ea73e5a-e8782ed1d2a5492e99b9bf6bd4b1597a
I20240501 08:14:33.074237 13090 tablet_copy_source_session.cc:205] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:33.076633 18004 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8782ed1d2a5492e99b9bf6bd4b1597a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:33.079190 18020 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.083393 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:33.082603 15366 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.086699 18004 tablet_copy_client.cc:783] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:33.086988 18004 tablet_copy_client.cc:647] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:33.092445 12671 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.092561 13070 raft_consensus.cc:1261] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer fa5b68f497b84b31ac35e5549b2af1ad: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.093257 12531 catalog_manager.cc:5484] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.097882 13447 tablet_service.cc:1514] Processing DeleteTablet for tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b with delete_type TABLET_DATA_TOMBSTONED (TS b17fe4a1f54c4655b85eb663fcff1b93 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:53394
I20240501 08:14:33.080842 18157 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.108568 18157 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.110237 18151 consensus_queue.cc:1035] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P fa5b68f497b84b31ac35e5549b2af1ad [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.110252 13335 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.114513 18004 tablet_copy_client.cc:523] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:33.116925 18020 tablet_bootstrap.cc:492] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap complete.
I20240501 08:14:33.117276 18020 ts_tablet_manager.cc:1367] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.720s user 0.062s sys 0.001s
I20240501 08:14:33.118109 18020 raft_consensus.cc:348] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.118443 18020 raft_consensus.cc:729] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: LEARNER
I20240501 08:14:33.118687 18020 consensus_queue.cc:260] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.119349 18020 ts_tablet_manager.cc:1398] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240501 08:14:33.120137 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-cab182a0e126421c986101ea19df3e97 received from {username='slave'} at 127.28.1.133:47379
I20240501 08:14:33.120270 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-cab182a0e126421c986101ea19df3e97 on tablet cab182a0e126421c986101ea19df3e97 with peer fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:33.123155 18004 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap starting.
I20240501 08:14:33.124727 12937 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.125697 18185 tablet_replica.cc:330] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: stopping tablet replica
I20240501 08:14:33.128196 18185 raft_consensus.cc:2229] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:33.128661 18185 raft_consensus.cc:2258] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:33.132238 18020 ts_tablet_manager.cc:902] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Initiating tablet copy from peer 56e1ccecd51040458c27911fa34a90a6 (127.28.1.129:45349)
I20240501 08:14:33.132213 12671 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.133179 13070 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.134522 18185 ts_tablet_manager.cc:1875] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:33.154649 15101 raft_consensus.cc:2932] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.157379 18185 ts_tablet_manager.cc:1888] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:33.160212 18020 tablet_copy_client.cc:308] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.129:45349
I20240501 08:14:33.162004 12694 tablet_copy_service.cc:140] P 56e1ccecd51040458c27911fa34a90a6: Received BeginTabletCopySession request for tablet cd1c9882eb034801aa59324fe069461e from peer fa5b68f497b84b31ac35e5549b2af1ad ({username='slave'} at 127.28.1.133:33761)
I20240501 08:14:33.162164 12694 tablet_copy_service.cc:161] P 56e1ccecd51040458c27911fa34a90a6: Beginning new tablet copy session on tablet cd1c9882eb034801aa59324fe069461e from peer fa5b68f497b84b31ac35e5549b2af1ad at {username='slave'} at 127.28.1.133:33761: session id = fa5b68f497b84b31ac35e5549b2af1ad-cd1c9882eb034801aa59324fe069461e
I20240501 08:14:33.165136 12694 tablet_copy_source_session.cc:205] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:33.170526 18020 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:33.173092 18185 log.cc:1196] T c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b P b17fe4a1f54c4655b85eb663fcff1b93: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b
I20240501 08:14:33.175208 12518 catalog_manager.cc:4830] TS b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): tablet c9cfc8eb09cf4eb1b8cf9de0ec1d1f5b (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:33.184917 13070 raft_consensus.cc:1203] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Deduplicated request from leader. Original: 1.8->[1.9-1.9] Dedup: 1.9->[]
I20240501 08:14:33.187932 15101 raft_consensus.cc:1203] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.8->[1.9-1.9] Dedup: 1.9->[]
I20240501 08:14:33.193264 12671 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.195760 18020 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:33.196074 18020 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd1c9882eb034801aa59324fe069461e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:33.196430 18096 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.196892 18096 tablet_bootstrap.cc:492] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap complete.
I20240501 08:14:33.197185 18096 ts_tablet_manager.cc:1367] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.462s user 0.052s sys 0.002s
I20240501 08:14:33.198365 18096 raft_consensus.cc:348] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.198774 18096 raft_consensus.cc:729] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: LEARNER
I20240501 08:14:33.199451 18020 tablet_copy_client.cc:783] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:33.199667 18020 tablet_copy_client.cc:647] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:33.199000 18096 consensus_queue.cc:260] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.200695 18096 ts_tablet_manager.cc:1398] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:33.201591 18020 tablet_copy_client.cc:523] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:33.201881 12959 tablet_copy_service.cc:342] P 5bf85222f4734b7885eb866efd27d5f2: Request end of tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-2b098b5bce924e6ea5897e463451b964 received from {username='slave'} at 127.28.1.133:48547
I20240501 08:14:33.202059 12959 tablet_copy_service.cc:433] P 5bf85222f4734b7885eb866efd27d5f2: ending tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-2b098b5bce924e6ea5897e463451b964 on tablet 2b098b5bce924e6ea5897e463451b964 with peer fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:33.205356 18020 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap starting.
I20240501 08:14:33.205489 15366 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.206715 15101 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.207698 18201 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.208437 18087 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.209666 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.210300 17890 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.213469 17890 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.218806 15101 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.223021 13201 raft_consensus.cc:1203] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:33.227918 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd1c9882eb034801aa59324fe069461e with cas_config_opid_index 13: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:33.228753 15366 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.230144 13070 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.234325 12528 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:33.246567 12918 tablet_service.cc:1514] Processing DeleteTablet for tablet cd1c9882eb034801aa59324fe069461e with delete_type TABLET_DATA_TOMBSTONED (TS 5bf85222f4734b7885eb866efd27d5f2 not found in new config with opid_index 14) from {username='slave'} at 127.0.0.1:44580
I20240501 08:14:33.250047 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.250763 18201 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.252490 13466 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.271288 18208 tablet_replica.cc:330] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: stopping tablet replica
I20240501 08:14:33.271593 18208 raft_consensus.cc:2229] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:33.272388 18004 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.272578 18208 raft_consensus.cc:2258] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:33.273367 18004 tablet_bootstrap.cc:492] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Bootstrap complete.
I20240501 08:14:33.273653 18208 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:33.273736 18004 ts_tablet_manager.cc:1367] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent bootstrapping tablet: real 0.151s user 0.044s sys 0.009s
I20240501 08:14:33.274721 18004 raft_consensus.cc:348] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:33.275079 18004 raft_consensus.cc:729] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3cd5ec97de8642dd8796fae63ea73e5a, State: Initialized, Role: LEARNER
I20240501 08:14:33.275405 18004 consensus_queue.cc:260] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
I20240501 08:14:33.276105 18004 ts_tablet_manager.cc:1398] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240501 08:14:33.277546 13090 tablet_copy_service.cc:342] P f59d8b4fd79049b6b40dcecd13d0be9d: Request end of tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-e8782ed1d2a5492e99b9bf6bd4b1597a received from {username='slave'} at 127.28.1.138:57959
I20240501 08:14:33.277693 13090 tablet_copy_service.cc:433] P f59d8b4fd79049b6b40dcecd13d0be9d: ending tablet copy session 3cd5ec97de8642dd8796fae63ea73e5a-e8782ed1d2a5492e99b9bf6bd4b1597a on tablet e8782ed1d2a5492e99b9bf6bd4b1597a with peer 3cd5ec97de8642dd8796fae63ea73e5a
W20240501 08:14:33.281903 12608 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 81686.3us
W20240501 08:14:33.282296 12608 consensus_peers.cc:485] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: SHUTDOWN. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:33.282343 18208 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.13
I20240501 08:14:33.282500 18208 log.cc:1196] T cd1c9882eb034801aa59324fe069461e P 5bf85222f4734b7885eb866efd27d5f2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-2/wal/wals/cd1c9882eb034801aa59324fe069461e
I20240501 08:14:33.283437 12518 catalog_manager.cc:4830] TS 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131:38689): tablet cd1c9882eb034801aa59324fe069461e (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:33.285180 15366 raft_consensus.cc:1203] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:33.286903 12802 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.287837 18201 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.289424 13335 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:33.289718 12802 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.292358 13070 raft_consensus.cc:1203] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:33.312680 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:33.315127 13335 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.333242 17923 raft_consensus.cc:1050] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:33.334553 17923 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:33.351066 15101 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.353312 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.356503 15237 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.358235 18075 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.390199 15368 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.393153 18020 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=13 overwritten=0 applied=13 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:33.394676 18020 tablet_bootstrap.cc:492] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Bootstrap complete.
I20240501 08:14:33.398156 18020 ts_tablet_manager.cc:1367] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent bootstrapping tablet: real 0.193s user 0.095s sys 0.021s
I20240501 08:14:33.399204 18020 raft_consensus.cc:348] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.399698 18020 raft_consensus.cc:729] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa5b68f497b84b31ac35e5549b2af1ad, State: Initialized, Role: LEARNER
I20240501 08:14:33.400043 18020 consensus_queue.cc:260] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 13, Last appended: 1.13, Last appended by leader: 13, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } attrs { replace: true } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:33.400400 18049 raft_consensus.cc:1050] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:33.400964 18020 ts_tablet_manager.cc:1398] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240501 08:14:33.401494 18049 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:33.402186 12694 tablet_copy_service.cc:342] P 56e1ccecd51040458c27911fa34a90a6: Request end of tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-cd1c9882eb034801aa59324fe069461e received from {username='slave'} at 127.28.1.133:33761
I20240501 08:14:33.402324 12694 tablet_copy_service.cc:433] P 56e1ccecd51040458c27911fa34a90a6: ending tablet copy session fa5b68f497b84b31ac35e5549b2af1ad-cd1c9882eb034801aa59324fe069461e on tablet cd1c9882eb034801aa59324fe069461e with peer fa5b68f497b84b31ac35e5549b2af1ad
I20240501 08:14:33.409526 15100 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.410188 18202 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.415364 15235 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.415994 13466 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.416831 18201 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.417349 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.417443 18201 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.428689 13201 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.429337 18202 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.434713 12802 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.434821 12673 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.435397 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.436061 18075 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.439446 13335 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.440811 13202 raft_consensus.cc:1203] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:33.441754 18202 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.452394 12937 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.452558 15236 raft_consensus.cc:1203] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240501 08:14:33.453122 18201 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.455387 12806 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.456354 18201 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.475334 13330 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.476078 18075 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.476557 13465 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.477730 18075 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.505077 17923 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.505412 18202 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.510228 13467 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.513710 12802 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.516338 15101 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.526530 13335 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.527189 13202 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.529314 13467 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.531713 13330 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.534719 12528 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.538141 12524 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:33.542037 15237 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.552937 15368 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.557381 12937 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.558540 12673 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.563650 13070 consensus_queue.cc:237] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:33.568605 13331 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.569157 18075 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.571285 15366 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.571655 15236 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.572924 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.573472 17570 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.578392 18075 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.582554 13331 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.586000 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e9b18b849c914fa5ba731892b7b88d6e with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:33.589751 13467 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.590476 18221 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.590268 12528 catalog_manager.cc:5484] T e9b18b849c914fa5ba731892b7b88d6e P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.592286 13465 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.599164 12802 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.602636 18222 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.604755 12803 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.610949 12806 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.622486 15235 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.626137 15083 tablet_service.cc:1514] Processing DeleteTablet for tablet e9b18b849c914fa5ba731892b7b88d6e with delete_type TABLET_DATA_TOMBSTONED (TS 8e309f48ce254660915fb9fc32649ab2 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:39520
I20240501 08:14:33.644738 12672 consensus_queue.cc:237] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } }
I20240501 08:14:33.648041 13201 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.648113 15101 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.648720 18207 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.649333 18207 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.650044 13466 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.650946 18207 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.651192 15101 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.656565 13330 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.657049 12803 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.657171 18207 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.658167 18202 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.658568 18201 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.661342 13330 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.662811 15101 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.663861 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:33.666240 13201 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.667745 13466 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.670363 12527 catalog_manager.cc:5484] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:33.683712 12803 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.696449 12671 raft_consensus.cc:1261] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:33.697563 18222 consensus_queue.cc:1035] T e9b18b849c914fa5ba731892b7b88d6e P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:33.681500 17962 raft_consensus.cc:1050] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER 8e309f48ce254660915fb9fc32649ab2 to VOTER
I20240501 08:14:33.702831 17962 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:33.708828 12673 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.715751 15235 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.719465 15100 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.719434 15366 raft_consensus.cc:2932] T e9b18b849c914fa5ba731892b7b88d6e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:33.719997 18250 tablet_replica.cc:330] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: stopping tablet replica
I20240501 08:14:33.720306 18250 raft_consensus.cc:2229] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:33.720642 18250 raft_consensus.cc:2258] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:33.721091 12934 raft_consensus.cc:1261] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.724742 18250 ts_tablet_manager.cc:1875] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:33.725343 18049 consensus_queue.cc:1035] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.726955 17570 raft_consensus.cc:1050] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 35cb9fadf231408598c3877837a6149e to VOTER
I20240501 08:14:33.728173 17570 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:33.728060 12934 raft_consensus.cc:2932] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } }
I20240501 08:14:33.731393 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.739514 15366 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.740116 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.741953 13330 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.742616 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.749547 13201 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.750578 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.762401 13070 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.763738 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.770349 13467 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.771046 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.782117 15366 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.782847 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240501 08:14:33.783272 18252 log.cc:924] Time spent T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: Append to log took a long time: real 0.065s user 0.000s sys 0.000s
I20240501 08:14:33.786954 13466 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.786983 15100 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.787463 18221 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.787875 18222 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.789419 12673 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.790062 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.802014 15217 tablet_service.cc:1514] Processing DeleteTablet for tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 with delete_type TABLET_DATA_TOMBSTONED (TS 3cd5ec97de8642dd8796fae63ea73e5a not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:44248
I20240501 08:14:33.809262 15235 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.814975 18245 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.817487 12673 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.818550 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.829041 12803 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:33.829586 12806 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.830480 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.832492 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:33.835882 18278 tablet_replica.cc:330] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: stopping tablet replica
I20240501 08:14:33.836258 17526 raft_consensus.cc:1050] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93: attempting to promote NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d to VOTER
I20240501 08:14:33.836565 18278 raft_consensus.cc:2229] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:33.837064 18278 raft_consensus.cc:2258] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:33.837388 17526 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:33.838796 18278 ts_tablet_manager.cc:1875] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:33.844069 17806 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.846818 13465 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.854348 13330 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.855108 17570 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.860546 12803 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.862248 13070 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.865126 13068 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.865443 15237 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.865885 17965 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.866535 18180 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.869702 18278 ts_tablet_manager.cc:1888] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:33.870693 13201 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.870870 12531 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.871258 18180 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.871587 18278 log.cc:1196] T 2ede6a5ca6654aceb2a24e1592fc1bc8 P 3cd5ec97de8642dd8796fae63ea73e5a: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal/wals/2ede6a5ca6654aceb2a24e1592fc1bc8
I20240501 08:14:33.876139 12518 catalog_manager.cc:4830] TS 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): tablet 2ede6a5ca6654aceb2a24e1592fc1bc8 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:33.882697 12934 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.882826 12806 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.883576 18180 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.884155 18180 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.898332 18290 raft_consensus.cc:1050] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d to VOTER
I20240501 08:14:33.897845 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.899271 15368 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.908581 13334 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:33.909301 13330 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.910754 17965 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.911458 17965 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:33.913491 12673 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:33.914502 18245 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.917171 12671 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.920540 15367 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.923594 12531 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:33.923508 15237 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.924336 15100 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.928568 13202 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Committing config change with OpId 1.14: config changed from index 13 to 14, VOTER 5bf85222f4734b7885eb866efd27d5f2 (127.28.1.131) evicted. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:33.935683 17526 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.935300 12803 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.941375 15235 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.944557 18282 raft_consensus.cc:1050] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad to VOTER
I20240501 08:14:33.943437 13070 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.947824 18282 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:33.949276 12934 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.950292 12524 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 reported cstate change: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:33.954614 18250 ts_tablet_manager.cc:1888] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:33.954823 18250 log.cc:1196] T e9b18b849c914fa5ba731892b7b88d6e P 8e309f48ce254660915fb9fc32649ab2: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-8/wal/wals/e9b18b849c914fa5ba731892b7b88d6e
I20240501 08:14:33.955832 12515 catalog_manager.cc:4830] TS 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137:37989): tablet e9b18b849c914fa5ba731892b7b88d6e (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:33.958206 18221 raft_consensus.cc:1050] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d: attempting to promote NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a to VOTER
I20240501 08:14:33.958201 15367 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:33.959337 18221 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:33.959983 18290 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:33.962558 18022 raft_consensus.cc:1013] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: attempt to promote peer f59d8b4fd79049b6b40dcecd13d0be9d: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240501 08:14:33.964581 13200 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:33.965723 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:33.966033 13068 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:33.968351 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:33.974229 18262 raft_consensus.cc:1050] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2: attempting to promote NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad to VOTER
I20240501 08:14:33.975478 18262 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:33.980540 13335 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:33.987346 13332 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240501 08:14:33.988642 13402 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.134:37699, user_credentials={real_user=slave}} blocked reactor thread for 78142.6us
I20240501 08:14:33.992465 12806 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.992617 12673 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.993295 17923 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.993487 15368 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.994227 17923 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.994675 15235 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 LEARNER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.994887 17570 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.995296 17923 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.996340 12936 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:33.996124 17806 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.007s
I20240501 08:14:33.996876 17923 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:33.994220 13330 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.011365 13466 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.013042 13467 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.014307 17570 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.016222 13201 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.017302 12802 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.018483 17570 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.019351 13467 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 35cb9fadf231408598c3877837a6149e (127.28.1.136) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.031379 18291 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.034701 17962 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.037458 13465 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.038718 13068 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } }
I20240501 08:14:34.044112 13335 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.044144 12671 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.044715 18245 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.045337 15367 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.046471 15100 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.046777 15236 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.047075 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.048272 18022 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.048545 12803 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.049024 18245 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.051446 17570 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.051954 17570 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.003s
I20240501 08:14:34.054988 18222 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.055610 12803 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.055909 15100 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.060305 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 795dd1b66844485dad769acd4ec2b16f with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.059922 13202 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.061954 12527 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.062259 12671 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.066277 15217 tablet_service.cc:1514] Processing DeleteTablet for tablet 795dd1b66844485dad769acd4ec2b16f with delete_type TABLET_DATA_TOMBSTONED (TS 3cd5ec97de8642dd8796fae63ea73e5a not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:44248
I20240501 08:14:34.066656 18278 tablet_replica.cc:330] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: stopping tablet replica
I20240501 08:14:34.066836 18278 raft_consensus.cc:2229] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.067090 18278 raft_consensus.cc:2258] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.070013 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.069396 13335 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.072724 15367 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.073863 13463 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.074553 18022 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.074858 13334 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:34.078675 18278 ts_tablet_manager.cc:1875] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.082124 13335 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.082794 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.085741 18278 ts_tablet_manager.cc:1888] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:34.086752 15366 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.087697 17749 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.088346 15100 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.090059 12937 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.089939 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.090624 18288 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.090814 12803 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.091563 13202 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.092099 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.092389 18288 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240501 08:14:34.095180 13269 consensus_peers.cc:485] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 -> Peer f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132:43293): Couldn't send request to peer f59d8b4fd79049b6b40dcecd13d0be9d. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:34.095746 18278 log.cc:1196] T 795dd1b66844485dad769acd4ec2b16f P 3cd5ec97de8642dd8796fae63ea73e5a: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-9/wal/wals/795dd1b66844485dad769acd4ec2b16f
I20240501 08:14:34.097149 12518 catalog_manager.cc:4830] TS 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): tablet 795dd1b66844485dad769acd4ec2b16f (table rebalance_test_table_0 [id=a98e619768904b4e88117186b0569b4e]) successfully deleted
I20240501 08:14:34.104354 12671 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.104931 18022 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.111040 12802 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.116779 13464 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.118019 13200 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.118526 18288 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.124397 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.125070 15366 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240501 08:14:34.125623 12871 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.28.1.136:37957, user_credentials={real_user=slave}} blocked reactor thread for 272489us
I20240501 08:14:34.125727 13068 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240501 08:14:34.126706 15366 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.128163 15103 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.130599 15101 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, 8e309f48ce254660915fb9fc32649ab2 (127.28.1.137) changed from NON_VOTER to VOTER. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.134241 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.134877 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.136476 13465 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.138509 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.142094 17923 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.143507 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240501 08:14:34.146986 13465 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } }
I20240501 08:14:34.153391 12671 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.153797 15103 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.154549 12805 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.154873 15369 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240501 08:14:34.155344 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.155598 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.155598 13202 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.156551 12525 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:34.157399 12805 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.157934 12671 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.167049 13335 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.178200 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.002s
I20240501 08:14:34.178531 17962 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240501 08:14:34.178970 13332 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { replace: true } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.189076 17923 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.194537 15368 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.197842 13463 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.198514 12934 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.198876 18277 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.199734 13070 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.200748 12527 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:34.201768 18277 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.202837 13330 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.205121 15235 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.208231 12805 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.208292 12936 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.211112 15369 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.212716 12671 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.201148 13069 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.214345 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.213503 17749 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.214668 12528 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.217133 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.234076 13467 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.239192 12806 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.243772 12673 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.246461 15100 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.247412 18290 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.252841 15235 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.258512 13465 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.260370 12802 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.260466 13201 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.266489 12522 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 reported cstate change: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.276866 18282 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.280382 12670 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.287325 15368 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.288300 15103 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.295172 13463 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.294838 12529 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 reported cstate change: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.286123 13331 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.297894 13068 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.298988 12672 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.300071 15367 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.308370 18201 raft_consensus.cc:1050] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6: attempting to promote NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad to VOTER
I20240501 08:14:34.309623 18201 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 14, Committed index: 14, Last appended: 1.14, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:34.312809 13464 consensus_queue.cc:237] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:34.317028 12937 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.319284 13068 consensus_queue.cc:237] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } }
I20240501 08:14:34.322222 12936 consensus_queue.cc:237] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:34.322549 15235 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.323652 12934 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.324414 18222 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.325142 18222 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.326174 15236 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.327051 13202 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 LEARNER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.327430 18262 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.327637 12803 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.328590 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.329183 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.330188 13201 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.330525 12805 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.330682 13068 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.331040 18262 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.331599 18290 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.332134 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.336944 12671 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.338183 13334 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.338527 15367 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.339001 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.339145 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.338944 13069 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.339500 17806 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.339697 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.340919 18207 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.341437 17806 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.344682 12805 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.350664 15101 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.350878 12669 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.351706 18201 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.354532 15103 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.360543 13332 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.361367 18222 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.364073 13466 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240501 08:14:34.364199 17923 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.365180 13332 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.366698 15235 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.367259 17526 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.367630 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240501 08:14:34.368310 13067 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.370720 18299 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.373986 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e8782ed1d2a5492e99b9bf6bd4b1597a with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.375345 12936 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.378989 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2b098b5bce924e6ea5897e463451b964 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.380671 13202 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.381175 15236 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.381933 15235 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.386543 12529 catalog_manager.cc:5484] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240501 08:14:34.387506 12529 catalog_manager.cc:5484] T e8782ed1d2a5492e99b9bf6bd4b1597a P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.390650 15368 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.391376 18299 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.393697 13447 tablet_service.cc:1514] Processing DeleteTablet for tablet 2b098b5bce924e6ea5897e463451b964 with delete_type TABLET_DATA_TOMBSTONED (TS b17fe4a1f54c4655b85eb663fcff1b93 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53394
I20240501 08:14:34.394656 18299 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.396306 15368 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.402134 12530 catalog_manager.cc:5484] T 85eb7c44993b4749b4b0a090d773bbef P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New cstate: current_term: 1 leader_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.406363 12803 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.406841 12936 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.406971 17526 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.408005 17526 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.409222 13181 tablet_service.cc:1514] Processing DeleteTablet for tablet 85eb7c44993b4749b4b0a090d773bbef with delete_type TABLET_DATA_TOMBSTONED (TS fa5b68f497b84b31ac35e5549b2af1ad not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:40618
I20240501 08:14:34.408901 12803 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.409619 12936 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.420533 18049 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.424858 13202 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.426154 13464 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.427553 12936 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.427563 13201 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.431758 13334 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.432057 12803 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.434885 15101 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.439008 12529 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.444438 13330 raft_consensus.cc:1261] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer b17fe4a1f54c4655b85eb663fcff1b93: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.445065 17526 consensus_queue.cc:1035] T 85eb7c44993b4749b4b0a090d773bbef P b17fe4a1f54c4655b85eb663fcff1b93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.444990 13070 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.446853 13333 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.458163 12672 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.458290 13462 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.459340 18245 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.460472 18245 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.462119 12672 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.463757 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 85eb7c44993b4749b4b0a090d773bbef with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.468413 18359 tablet_replica.cc:330] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: stopping tablet replica
I20240501 08:14:34.468660 18359 raft_consensus.cc:2229] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.469008 18359 raft_consensus.cc:2258] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.469389 13462 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.476187 18359 ts_tablet_manager.cc:1875] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.480448 18356 tablet_replica.cc:330] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: stopping tablet replica
I20240501 08:14:34.480775 18356 raft_consensus.cc:2229] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.480630 15235 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P 3cd5ec97de8642dd8796fae63ea73e5a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.481227 18356 raft_consensus.cc:2258] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.482414 18356 ts_tablet_manager.cc:1875] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.492580 15367 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.493407 15349 tablet_service.cc:1514] Processing DeleteTablet for tablet e8782ed1d2a5492e99b9bf6bd4b1597a with delete_type TABLET_DATA_TOMBSTONED (TS 35cb9fadf231408598c3877837a6149e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:41086
I20240501 08:14:34.498808 12672 consensus_queue.cc:237] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 15, Committed index: 15, Last appended: 1.15, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:34.505786 13201 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.506346 15101 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.509754 13467 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.511041 18362 tablet_replica.cc:330] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: stopping tablet replica
I20240501 08:14:34.511734 12802 raft_consensus.cc:1261] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.513450 18245 consensus_queue.cc:1035] T e8782ed1d2a5492e99b9bf6bd4b1597a P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.515203 18362 raft_consensus.cc:2229] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.515445 12802 raft_consensus.cc:2932] T e8782ed1d2a5492e99b9bf6bd4b1597a P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } }
I20240501 08:14:34.516570 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240501 08:14:34.517156 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240501 08:14:34.499843 13069 raft_consensus.cc:2932] T 85eb7c44993b4749b4b0a090d773bbef P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.522583 13068 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.528390 15368 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.528831 13335 raft_consensus.cc:1261] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240501 08:14:34.529044 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240501 08:14:34.529502 18362 raft_consensus.cc:2258] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.529727 18290 consensus_queue.cc:1035] T 2b098b5bce924e6ea5897e463451b964 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240501 08:14:34.532356 12803 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.532474 18356 ts_tablet_manager.cc:1888] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:34.532632 18356 log.cc:1196] T 2b098b5bce924e6ea5897e463451b964 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/2b098b5bce924e6ea5897e463451b964
I20240501 08:14:34.533716 12518 catalog_manager.cc:4830] TS b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): tablet 2b098b5bce924e6ea5897e463451b964 (table rebalance_test_table_2 [id=a0bfa985a3fd478eb83b407df15b7ffb]) successfully deleted
I20240501 08:14:34.533758 18207 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.536898 13331 raft_consensus.cc:2932] T 2b098b5bce924e6ea5897e463451b964 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.539999 18359 ts_tablet_manager.cc:1888] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:34.541399 18362 ts_tablet_manager.cc:1875] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.541441 18359 log.cc:1196] T 85eb7c44993b4749b4b0a090d773bbef P fa5b68f497b84b31ac35e5549b2af1ad: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-4/wal/wals/85eb7c44993b4749b4b0a090d773bbef
I20240501 08:14:34.539801 13201 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } } }
I20240501 08:14:34.542621 13335 consensus_queue.cc:237] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } }
I20240501 08:14:34.544337 12517 catalog_manager.cc:4830] TS fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): tablet 85eb7c44993b4749b4b0a090d773bbef (table rebalance_test_table_1 [id=f68539f804fc43ab8704269e9b7439c6]) successfully deleted
I20240501 08:14:34.544768 12516 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd1c9882eb034801aa59324fe069461e with cas_config_opid_index 15: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.546615 15368 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.548386 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.553860 15101 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.554756 18277 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.562343 12936 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.563057 18279 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.565356 18363 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240501 08:14:34.566447 18279 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.571193 13202 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.571995 18277 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.574396 13068 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.575122 18288 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.576895 13068 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.579085 12803 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.581511 12673 raft_consensus.cc:1261] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:34.582741 18277 consensus_queue.cc:1035] T cab182a0e126421c986101ea19df3e97 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:34.584470 12669 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.588119 15101 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.590032 13331 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.590193 13070 raft_consensus.cc:1261] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240501 08:14:34.590801 12530 catalog_manager.cc:5484] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 reported cstate change: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:34.590719 18049 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.002s
I20240501 08:14:34.594118 18363 consensus_queue.cc:1035] T cd1c9882eb034801aa59324fe069461e P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240501 08:14:34.596160 13334 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.595835 13201 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.598228 13070 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.592947 12936 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.601238 13202 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.610883 12527 catalog_manager.cc:5484] T cab182a0e126421c986101ea19df3e97 P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.614521 12517 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cab182a0e126421c986101ea19df3e97 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.610037 15367 raft_consensus.cc:2932] T cd1c9882eb034801aa59324fe069461e P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.620494 15101 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.646453 13447 tablet_service.cc:1514] Processing DeleteTablet for tablet cd1c9882eb034801aa59324fe069461e with delete_type TABLET_DATA_TOMBSTONED (TS b17fe4a1f54c4655b85eb663fcff1b93 not found in new config with opid_index 16) from {username='slave'} at 127.0.0.1:53394
I20240501 08:14:34.646761 13445 tablet_service.cc:1514] Processing DeleteTablet for tablet cab182a0e126421c986101ea19df3e97 with delete_type TABLET_DATA_TOMBSTONED (TS b17fe4a1f54c4655b85eb663fcff1b93 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:53394
I20240501 08:14:34.646976 18356 tablet_replica.cc:330] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: stopping tablet replica
I20240501 08:14:34.647182 18356 raft_consensus.cc:2229] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.647836 18356 raft_consensus.cc:2258] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.655350 18356 ts_tablet_manager.cc:1875] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.636447 12936 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:34.666910 18356 ts_tablet_manager.cc:1888] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.15
I20240501 08:14:34.620532 15368 raft_consensus.cc:2932] T cab182a0e126421c986101ea19df3e97 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: false } } }
I20240501 08:14:34.676609 18374 ts_tablet_manager.cc:902] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Initiating tablet copy from peer c58c4517fd6941f19a9e75d1e48e8b29 (127.28.1.134:37699)
I20240501 08:14:34.679021 18356 log.cc:1196] T cd1c9882eb034801aa59324fe069461e P b17fe4a1f54c4655b85eb663fcff1b93: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/cd1c9882eb034801aa59324fe069461e
I20240501 08:14:34.680086 15101 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.680819 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.681964 15368 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.682500 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.687211 18356 tablet_replica.cc:330] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: stopping tablet replica
I20240501 08:14:34.687196 12518 catalog_manager.cc:4830] TS b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): tablet cd1c9882eb034801aa59324fe069461e (table rebalance_test_table_3 [id=ce0e75c51af44f0ca2dfd10a429d162a]) successfully deleted
I20240501 08:14:34.687485 18356 raft_consensus.cc:2229] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.688694 18356 raft_consensus.cc:2258] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.689934 18356 ts_tablet_manager.cc:1875] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.694557 18362 ts_tablet_manager.cc:1888] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240501 08:14:34.695834 13334 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.696569 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.702306 13202 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.703006 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.705991 12669 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.705897 17806 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.706521 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.708345 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.709947 12669 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.711171 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1226795e4f534366874b23270d086334 with cas_config_opid_index 11: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:34.713737 12528 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad reported cstate change: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.716656 13069 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.717691 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.719491 13068 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.723011 13445 tablet_service.cc:1514] Processing DeleteTablet for tablet 1226795e4f534366874b23270d086334 with delete_type TABLET_DATA_TOMBSTONED (TS b17fe4a1f54c4655b85eb663fcff1b93 not found in new config with opid_index 12) from {username='slave'} at 127.0.0.1:53394
I20240501 08:14:34.728145 12803 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:34.729717 18022 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:34.730409 15101 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.730607 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.731423 12803 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.740958 18356 ts_tablet_manager.cc:1888] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:34.746496 18356 log.cc:1196] T cab182a0e126421c986101ea19df3e97 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/cab182a0e126421c986101ea19df3e97
I20240501 08:14:34.747581 12518 catalog_manager.cc:4830] TS b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): tablet cab182a0e126421c986101ea19df3e97 (table rebalance_test_table_5 [id=5066668b79c844a29804267811e2b911]) successfully deleted
I20240501 08:14:34.748720 18356 tablet_replica.cc:330] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: stopping tablet replica
I20240501 08:14:34.749716 18356 raft_consensus.cc:2229] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:34.728910 15367 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, VOTER b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135) evicted. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:34.750711 18362 log.cc:1196] T e8782ed1d2a5492e99b9bf6bd4b1597a P 35cb9fadf231408598c3877837a6149e: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/wals/e8782ed1d2a5492e99b9bf6bd4b1597a
I20240501 08:14:34.751181 18356 raft_consensus.cc:2258] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:34.752606 18356 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:34.754127 12515 catalog_manager.cc:4830] TS 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): tablet e8782ed1d2a5492e99b9bf6bd4b1597a (table rebalance_test_table_4 [id=1bb81af95c724560a6fbba042e913f54]) successfully deleted
I20240501 08:14:34.785045 18356 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.11
I20240501 08:14:34.788551 18356 log.cc:1196] T 1226795e4f534366874b23270d086334 P b17fe4a1f54c4655b85eb663fcff1b93: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-6/wal/wals/1226795e4f534366874b23270d086334
I20240501 08:14:34.790124 12518 catalog_manager.cc:4830] TS b17fe4a1f54c4655b85eb663fcff1b93 (127.28.1.135:43031): tablet 1226795e4f534366874b23270d086334 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:34.801463 18374 tablet_copy_client.cc:308] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Beginning tablet copy session from remote peer at address 127.28.1.134:37699
I20240501 08:14:34.802385 13355 tablet_copy_service.cc:140] P c58c4517fd6941f19a9e75d1e48e8b29: Received BeginTabletCopySession request for tablet b6828130a760491eb21fd3d346219748 from peer f59d8b4fd79049b6b40dcecd13d0be9d ({username='slave'} at 127.28.1.132:50257)
I20240501 08:14:34.802552 13355 tablet_copy_service.cc:161] P c58c4517fd6941f19a9e75d1e48e8b29: Beginning new tablet copy session on tablet b6828130a760491eb21fd3d346219748 from peer f59d8b4fd79049b6b40dcecd13d0be9d at {username='slave'} at 127.28.1.132:50257: session id = f59d8b4fd79049b6b40dcecd13d0be9d-b6828130a760491eb21fd3d346219748
I20240501 08:14:34.805143 13355 tablet_copy_source_session.cc:205] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: Tablet Copy: opened 0 blocks and 1 log segments
I20240501 08:14:34.806345 18374 ts_tablet_manager.cc:1875] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240501 08:14:34.815902 18374 ts_tablet_manager.cc:1888] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240501 08:14:34.816339 18374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6828130a760491eb21fd3d346219748. 1 dirs total, 0 dirs full, 0 dirs failed
I20240501 08:14:34.819908 18374 tablet_copy_client.cc:783] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 0 data blocks...
I20240501 08:14:34.820423 18374 tablet_copy_client.cc:647] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Starting download of 1 WAL segments...
I20240501 08:14:34.822347 18374 tablet_copy_client.cc:523] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240501 08:14:34.826179 18374 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap starting.
I20240501 08:14:34.909087 18374 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240501 08:14:34.909705 18374 tablet_bootstrap.cc:492] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Bootstrap complete.
I20240501 08:14:34.910179 18374 ts_tablet_manager.cc:1367] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent bootstrapping tablet: real 0.084s user 0.055s sys 0.012s
I20240501 08:14:34.911657 18374 raft_consensus.cc:348] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:34.912146 18374 raft_consensus.cc:729] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f59d8b4fd79049b6b40dcecd13d0be9d, State: Initialized, Role: LEARNER
I20240501 08:14:34.914256 18374 consensus_queue.cc:260] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: NON_VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: true } }
I20240501 08:14:34.915215 18374 ts_tablet_manager.cc:1398] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240501 08:14:34.916504 13355 tablet_copy_service.cc:342] P c58c4517fd6941f19a9e75d1e48e8b29: Request end of tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-b6828130a760491eb21fd3d346219748 received from {username='slave'} at 127.28.1.132:50257
I20240501 08:14:34.916666 13355 tablet_copy_service.cc:433] P c58c4517fd6941f19a9e75d1e48e8b29: ending tablet copy session f59d8b4fd79049b6b40dcecd13d0be9d-b6828130a760491eb21fd3d346219748 on tablet b6828130a760491eb21fd3d346219748 with peer f59d8b4fd79049b6b40dcecd13d0be9d
I20240501 08:14:34.998191 12786 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:35.039371 12651 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:35.061790 15349 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:35.086916 12918 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:35.110186 15083 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:35.118844 13315 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:35.126935 13049 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240501 08:14:35.185871 15217 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240501 08:14:35.205412 13069 raft_consensus.cc:1203] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240501 08:14:35.218578 12936 consensus_queue.cc:237] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:35.224710 13445 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240501 08:14:35.228497 15101 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.229350 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.234727 13069 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.235366 12669 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.236147 18305 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.236799 18305 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.246673 13201 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.248632 13331 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.251417 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.252095 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.258492 18022 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.259059 12669 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.262450 12518 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1226795e4f534366874b23270d086334 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240501 08:14:35.265966 13069 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.271399 13334 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.277993 13181 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240501 08:14:35.277858 12528 catalog_manager.cc:5484] T 1226795e4f534366874b23270d086334 P c58c4517fd6941f19a9e75d1e48e8b29 reported cstate change: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New cstate: current_term: 1 leader_uuid: "5bf85222f4734b7885eb866efd27d5f2" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.286284 12803 raft_consensus.cc:1261] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 5bf85222f4734b7885eb866efd27d5f2: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240501 08:14:35.285771 15101 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.287011 18262 consensus_queue.cc:1035] T 1226795e4f534366874b23270d086334 P 5bf85222f4734b7885eb866efd27d5f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240501 08:14:35.288362 13201 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.293895 12803 raft_consensus.cc:2932] T 1226795e4f534366874b23270d086334 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 35cb9fadf231408598c3877837a6149e (127.28.1.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.380080 15349 tablet_service.cc:1514] Processing DeleteTablet for tablet 1226795e4f534366874b23270d086334 with delete_type TABLET_DATA_TOMBSTONED (TS 35cb9fadf231408598c3877837a6149e not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:41086
I20240501 08:14:35.393347 18476 tablet_replica.cc:330] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: stopping tablet replica
I20240501 08:14:35.393738 18476 raft_consensus.cc:2229] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240501 08:14:35.394217 18476 raft_consensus.cc:2258] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240501 08:14:35.400678 18476 ts_tablet_manager.cc:1875] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240501 08:14:35.408239 18476 ts_tablet_manager.cc:1888] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240501 08:14:35.408484 18476 log.cc:1196] T 1226795e4f534366874b23270d086334 P 35cb9fadf231408598c3877837a6149e: Deleting WAL directory at /tmp/dist-test-taskVWMLaH/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714551041925391-28678-0/raft_consensus-itest-cluster/ts-7/wal/wals/1226795e4f534366874b23270d086334
I20240501 08:14:35.409435 12515 catalog_manager.cc:4830] TS 35cb9fadf231408598c3877837a6149e (127.28.1.136:37957): tablet 1226795e4f534366874b23270d086334 (table rebalance_test_table_6 [id=6a106bd4d226436cb226a61be64eaada]) successfully deleted
I20240501 08:14:35.426020 13069 consensus_queue.cc:237] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } }
I20240501 08:14:35.444343 13334 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.444617 13467 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.445369 17570 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:35.445709 15101 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.446172 15367 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.446808 18222 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:35.447381 12669 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.447376 18222 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
W20240501 08:14:35.447748 13005 consensus_peers.cc:485] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d -> Peer fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133:42529): Couldn't send request to peer fa5b68f497b84b31ac35e5549b2af1ad. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:35.447885 18222 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.004s
I20240501 08:14:35.451845 18245 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:35.456079 12803 raft_consensus.cc:1261] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer f59d8b4fd79049b6b40dcecd13d0be9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240501 08:14:35.456813 18245 consensus_queue.cc:1035] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240501 08:14:35.475422 18245 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.477955 15367 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.482964 13334 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.487864 12669 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 56e1ccecd51040458c27911fa34a90a6 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.491127 13462 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.492937 12802 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.496209 15100 raft_consensus.cc:2932] T 795dd1b66844485dad769acd4ec2b16f P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } } }
I20240501 08:14:35.498431 12527 catalog_manager.cc:5484] T 795dd1b66844485dad769acd4ec2b16f P f59d8b4fd79049b6b40dcecd13d0be9d reported cstate change: config changed from index 13 to 14, NON_VOTER fa5b68f497b84b31ac35e5549b2af1ad (127.28.1.133) added. New cstate: current_term: 1 leader_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: NON_VOTER last_known_addr { host: "127.28.1.133" port: 42529 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240501 08:14:35.542095 12669 consensus_queue.cc:237] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } }
W20240501 08:14:35.549822 12609 consensus_peers.cc:485] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 -> Peer 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138:40307): Couldn't send request to peer 3cd5ec97de8642dd8796fae63ea73e5a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 70ff1e2f7acd4e0784fef633bd0fd64d. This is attempt 1: this message will repeat every 5th retry.
I20240501 08:14:35.552729 15367 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.553313 13462 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.554175 13201 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.557349 18363 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.559623 18207 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.560216 18207 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.578495 13334 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.578490 12936 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.579499 12802 raft_consensus.cc:1261] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer 56e1ccecd51040458c27911fa34a90a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240501 08:14:35.580243 18049 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.582219 18363 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.582912 18207 consensus_queue.cc:1035] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240501 08:14:35.589036 18207 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 56e1ccecd51040458c27911fa34a90a6 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.595520 15367 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.602792 12522 catalog_manager.cc:5484] T 70ff1e2f7acd4e0784fef633bd0fd64d P 35cb9fadf231408598c3877837a6149e reported cstate change: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New cstate: current_term: 1 leader_uuid: "56e1ccecd51040458c27911fa34a90a6" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.604452 13201 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.604837 13334 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.611649 13462 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.621650 12937 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.626464 12802 raft_consensus.cc:2932] T 70ff1e2f7acd4e0784fef633bd0fd64d P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 3cd5ec97de8642dd8796fae63ea73e5a (127.28.1.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "56e1ccecd51040458c27911fa34a90a6" member_type: VOTER last_known_addr { host: "127.28.1.129" port: 45349 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false replace: true } } peers { permanent_uuid: "3cd5ec97de8642dd8796fae63ea73e5a" member_type: NON_VOTER last_known_addr { host: "127.28.1.138" port: 40307 } attrs { promote: true } } }
I20240501 08:14:35.632828 18350 raft_consensus.cc:1050] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29: attempting to promote NON_VOTER f59d8b4fd79049b6b40dcecd13d0be9d to VOTER
I20240501 08:14:35.634069 18350 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } }
I20240501 08:14:35.644757 15367 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.645041 12802 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.645594 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.646469 12937 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.647464 15100 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.649075 17749 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.649725 17749 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.650251 17749 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.655310 13069 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 LEARNER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.656018 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.669303 13462 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.669340 13201 raft_consensus.cc:1261] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Refusing update from remote peer c58c4517fd6941f19a9e75d1e48e8b29: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240501 08:14:35.669963 18279 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.670570 18350 consensus_queue.cc:1035] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240501 08:14:35.677413 17749 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.679905 12937 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.686995 12527 catalog_manager.cc:5484] T b6828130a760491eb21fd3d346219748 P 5bf85222f4734b7885eb866efd27d5f2 reported cstate change: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.689798 12803 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 1be03ffeb3384b42bfee6ffaf3f451c9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.703121 13462 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P b17fe4a1f54c4655b85eb663fcff1b93 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.706533 13069 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P f59d8b4fd79049b6b40dcecd13d0be9d [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.707862 13201 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P fa5b68f497b84b31ac35e5549b2af1ad [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.715919 15101 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 8e309f48ce254660915fb9fc32649ab2 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.719334 15367 raft_consensus.cc:2932] T b6828130a760491eb21fd3d346219748 P 35cb9fadf231408598c3877837a6149e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, f59d8b4fd79049b6b40dcecd13d0be9d (127.28.1.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "b17fe4a1f54c4655b85eb663fcff1b93" member_type: VOTER last_known_addr { host: "127.28.1.135" port: 43031 } attrs { replace: true } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137" port: 37989 } attrs { promote: false } } peers { permanent_uuid: "35cb9fadf231408598c3877837a6149e" member_type: VOTER last_known_addr { host: "127.28.1.136" port: 37957 } attrs { promote: false } } peers { permanent_uuid: "f59d8b4fd79049b6b40dcecd13d0be9d" member_type: VOTER last_known_addr { host: "127.28.1.132" port: 43293 } attrs { promote: false } } }
I20240501 08:14:35.723158 13331 consensus_queue.cc:237] T b6828130a760491eb21fd3d346219748 P c58c4517fd6941f19a9e75d1e48e8b29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c58c4517fd6941f19a9e75d1e48e8b29" member_type: VOTER last_known_addr { host: "127.28.1.134" port: 37699 } } peers { permanent_uuid: "fa5b68f497b84b31ac35e5549b2af1ad" member_type: VOTER last_known_addr { host: "127.28.1.133" port: 42529 } } peers { permanent_uuid: "1be03ffeb3384b42bfee6ffaf3f451c9" member_type: VOTER last_known_addr { host: "127.28.1.130" port: 45135 } } peers { permanent_uuid: "5bf85222f4734b7885eb866efd27d5f2" member_type: VOTER last_known_addr { host: "127.28.1.131" port: 38689 } } peers { permanent_uuid: "8e309f48ce254660915fb9fc32649ab2" member_type: VOTER last_known_addr { host: "127.28.1.137"